diff --git a/.changeset/25582-import-dotted-compact-day-option-value-case.md b/.changeset/25582-import-dotted-compact-day-option-value-case.md new file mode 100644 index 00000000000..cb0eadfed72 --- /dev/null +++ b/.changeset/25582-import-dotted-compact-day-option-value-case.md @@ -0,0 +1,15 @@ +--- +'@objectstack/core': patch +--- + +fix(core): `POST /api/v1/data/:object/import` reads a dotted (`2011.07.01`) or compact (`20090615`) year-first day in a `date` or `datetime` cell, and matches a select / multiselect cell against the option values case-insensitively + +Clause-②: yes (widening: two date-cell spellings and an option value in another case, each refused before with `invalid_date` / `invalid_option`, are now read; nothing read before is refused or stored differently) +Metadata-protocol: none + +Two cell shapes a person reads without hesitation used to fail the whole row. Both are now read by the import's cell reader (`parseDateCell`, `matchOption`), on the commit and on the dry run: + +- **A year-first day, dotted or compact.** `YYYY.MM.DD` (`2011.07.01`, the dotted date of Chinese ledgers) and `YYYYMMDD` (`20090615`, the compact date of ERP exports) are stored as `2011-07-01` / `2009-06-15` in a `date` cell, and as midnight UTC in a `datetime` cell, as `2026/7/15` already is. The day must exist: `20090230` and `2009.02.30` stay `invalid_date`, never rolled over. No other form is widened: a one-digit month or day (`2011.7.1`), a clock after either spelling, a two-digit year, a day-first dotted date (`01.07.2011`), six or seven digits, a `time` cell and a JSON number `20090615` stay refused. An eight-digit cell on a `number` or `text` column is read exactly as before. +- **An option value in another case.** A select / multiselect cell `CN` matches the declared option value `cn` and stores `cn` — the declared value, never the cell's spelling — as labels already matched in any case. A cell that matched an exact value or a label before stores what it stored: the value pass runs after both. A cell naming, by case alone, two declared values that differ only by case (`cn` and `CN`, which the spec's lowercase option value refuses but a programmatically registered object can carry) is not resolved by guessing; it gets `invalid_option` as before. + +Every cell error of a row is still reported as the first one only; that half of the same request is not in this change. diff --git a/packages/core/src/utils/import-coerce.test.ts b/packages/core/src/utils/import-coerce.test.ts index 5dbc64bffa9..53c45663e13 100644 --- a/packages/core/src/utils/import-coerce.test.ts +++ b/packages/core/src/utils/import-coerce.test.ts @@ -212,6 +212,64 @@ describe('[#20534] parseDateCell — ISO 8601, the export shape or a year-first }); }); +/** + * [#25582, maintainer ruling 6377844879 letter A] Two more year-first spellings + * of a bare day: `YYYY.MM.DD` and `YYYYMMDD`, on a day that exists, for a + * `date` and a `datetime` — and no other form is widened. + */ +describe('[#25582] parseDateCell — the dotted and compact year-first day', () => { + it.each([ + ['2011.07.01', 'date', '2011-07-01'], + ['20090615', 'date', '2009-06-15'], + [' 2011.07.01 ', 'date', '2011-07-01'], + ['2028.02.29', 'date', '2028-02-29'], + ['20280229', 'date', '2028-02-29'], + ['0500.01.01', 'date', '0500-01-01'], + // A `datetime` reads the bare day as midnight UTC, as it reads `2026/7/15`. + ['2011.07.01', 'datetime', '2011-07-01T00:00:00.000Z'], + ['20090615', 'datetime', '2009-06-15T00:00:00.000Z'], + ] as const)('reads %j as a %s cell as %j', (cell, kind, stored) => { + expect(parseDateCell(cell, kind)).toBe(stored); + }); + + it.each([ + // A day that does not exist — refused, never rolled over. + ['20090230', 'date'], ['2009.02.30', 'date'], ['20260229', 'datetime'], + ['20091301', 'date'], ['20090100', 'date'], ['2009.13.01', 'datetime'], + // No other form is widened. + ['2011.7.1', 'date'], ['2011.07.01 9:00', 'datetime'], ['2011.07.01T09:00', 'datetime'], + ['11.07.01', 'date'], ['01.07.2011', 'date'], ['2011.07-01', 'date'], + ['200961', 'date'], ['2009615', 'date'], ['200906150', 'date'], ['20090615 9:00', 'datetime'], + // A `time` holds no day, so neither spelling is a `time` cell. + ['2011.07.01', 'time'], ['20090615', 'time'], + ] as const)('refuses %j as a %s cell', (cell, kind) => { + expect(parseDateCell(cell, kind)).toBeUndefined(); + }); + + it('reads the compact day only from a text cell, never from a number', () => { + expect(parseDateCell(20090615, 'date')).toBeUndefined(); + expect(parseDateCell(20090615, 'datetime')).toBeUndefined(); + }); + + it('leaves an eight-digit cell on a number or text field as it was', async () => { + const metaMap = new Map([ + ['n', { name: 'n', type: 'number' } as ExportFieldMeta], + ['code', { name: 'code', type: 'text' } as ExportFieldMeta], + ['d', { name: 'd', type: 'date' } as ExportFieldMeta], + ]); + const { data, errors } = await coerceRow({ n: '20090615', code: '20090615', d: '20090615' }, metaMap, {}); + expect(errors).toEqual([]); + expect(data).toEqual({ n: 20090615, code: '20090615', d: '2009-06-15' }); + }); + + it('fails the row with invalid_date for a compact day that does not exist', async () => { + const metaMap = new Map([['d', { name: 'd', type: 'date' } as ExportFieldMeta]]); + const { data, errors } = await coerceRow({ d: '20090230' }, metaMap, {}); + expect(data).toEqual({}); + expect(errors.map((e) => [e.field, e.code])).toEqual([['d', 'invalid_date']]); + }); +}); + describe('matchOption', () => { const options = [{ label: '高', value: 'high' }, { label: '低', value: 'low' }]; it('matches by option value (code)', () => { @@ -224,6 +282,31 @@ describe('matchOption', () => { it('returns undefined when nothing matches', () => { expect(matchOption('medium', options)).toBeUndefined(); }); + + // [#25582, maintainer ruling 6377844879 letter A] Values match case-insensitively too. + it('[#25582] matches by option value case-insensitively and stores the declared value', () => { + const countries = [{ label: '中国', value: 'cn' }, { label: '美国', value: 'us' }]; + expect(matchOption('CN', countries)).toBe('cn'); + expect(matchOption(' Us ', countries)).toBe('us'); + expect(matchOption('HIGH', options)).toBe('high'); + // Not a substring or prefix match. + expect(matchOption('CHN', countries)).toBeUndefined(); + }); + it('[#25582] keeps a label match ahead of the case-insensitive value match', () => { + // `CN` is this option's label and, by case alone, the other option's value: + // the label match it had before the value pass existed still wins. + expect(matchOption('CN', [{ label: 'Other', value: 'cn' }, { label: 'cn', value: 'china' }])).toBe('china'); + }); + it('[#25582] refuses, rather than guesses, a cell naming two values that differ only by case', () => { + const pair = [{ label: 'Lower', value: 'cn' }, { label: 'Upper', value: 'CN' }]; + // An exact value still matches exactly. + expect(matchOption('cn', pair)).toBe('cn'); + expect(matchOption('CN', pair)).toBe('CN'); + // `Cn` names both by case alone: no match, as before this pass existed. + expect(matchOption('Cn', pair)).toBeUndefined(); + // The same value declared twice is one value, not two. + expect(matchOption('CN', [{ label: 'a', value: 'cn' }, { label: 'b', value: 'cn' }])).toBe('cn'); + }); }); describe('splitMulti', () => { @@ -401,6 +484,28 @@ describe('coerceRow', () => { expect(data).toEqual({}); }); + it('[#25582] matches select and multiselect option values case-insensitively, storing the declared value', async () => { + const metaMap = meta({ + country: { type: 'select', options: [{ label: '中国', value: 'cn' }, { label: '美国', value: 'us' }] }, + markets: { type: 'multiselect', options: [{ label: '中国', value: 'cn' }, { label: '美国', value: 'us' }, { label: '日本', value: 'jp' }] }, + regions: { type: 'select', multiple: true, options: [{ label: '东部', value: 'east' }, { label: '西部', value: 'west' }] }, + }); + const { data, errors, keptOptionValues } = await coerceRow( + { country: 'CN', markets: 'CN; Us、日本', regions: 'EAST,west' }, + metaMap, + {}, + ); + expect(errors).toEqual([]); + expect(data).toEqual({ country: 'cn', markets: ['cn', 'us', 'jp'], regions: ['east', 'west'] }); + // Matched, not kept: the write carries no admission for them. + expect(keptOptionValues).toEqual({}); + + // A part that matches no value or label still fails, naming that part. + const bad = await coerceRow({ markets: 'CN; DE' }, metaMap, {}); + expect(bad.errors.map((e) => [e.field, e.code])).toEqual([['markets', 'invalid_option']]); + expect(bad.errors[0].message).toContain('DE'); + }); + it('honours createMissingOptions by keeping the raw select value', async () => { const metaMap = meta({ s: { type: 'select', options: [{ label: 'A', value: 'a' }] } }); const strict = await coerceRow({ s: 'zzz' }, metaMap, {}); diff --git a/packages/core/src/utils/import-coerce.ts b/packages/core/src/utils/import-coerce.ts index 0ea300c6a7e..60af90a9304 100644 --- a/packages/core/src/utils/import-coerce.ts +++ b/packages/core/src/utils/import-coerce.ts @@ -335,8 +335,9 @@ function readTimeOfDayCell(s: string): string | undefined { * stored as `2026-07-15T14:00Z` on a New York host and `…T02:00Z` on a * Shanghai one, a `date` a day apart) and reads `07/08/2026` month-first. A * space before a zone (`2026-07-15 10:00Z`) is refused as the write door - * refuses it. The one other text shape read at all is the year-first date, - * {@link YEAR_FIRST_CELL}. + * refuses it. The other text shapes read at all are the year-first date, + * {@link YEAR_FIRST_CELL}, and the year-first day spellings + * {@link YEAR_FIRST_DOTTED_DAY} and {@link YEAR_FIRST_COMPACT_DAY}. */ const ISO_TEMPORAL_CELL = /^(\d{4})-(\d{2})-(\d{2})(?:(T| )(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d+))?)?(Z|[+-]\d{2}:?\d{2})?)?$/; @@ -450,6 +451,44 @@ function readYearFirstCell(s: string): IsoTemporalCell | undefined { return { day, wall }; } +/** + * [#25582, maintainer ruling 6377844879 letter A] Two more year-first spellings + * of a bare calendar day, after trimming, each with a four-digit year and a + * two-digit month and day and nothing else in the cell: + * + * - `YYYY.MM.DD` (`2011.07.01`) — the dotted date written in Chinese + * correspondence and ledgers; + * - `YYYYMMDD` (`20090615`) — the compact date an ERP export writes. Read only + * from a TEXT cell: a number is never a temporal cell ({@link outsideYears}), + * so a JSON `20090615` stays refused. An xlsx cell holding that number reaches + * here as the text it shows (`import-prepare.ts`), so it is read. + * + * The year comes first and every part has a fixed width, so there is no field + * order to guess. The ruling widens these two forms and no other: a one-digit + * month or day (`2011.7.1`), a clock after either (`2011.07.01 9:00`), a + * two-digit year (`11.07.01`), six or seven digits (`200961`) and a day-first + * dotted date (`01.07.2011`) stay refused. The day must exist + * ({@link namesRealCalendarDay}): `20090230` is refused, never rolled over. + * Read for a `date` and a `datetime` (midnight UTC, as a bare `2026/7/15` is + * read there), never for a `time`, which holds no day. + */ +const YEAR_FIRST_DOTTED_DAY = /^(\d{4})\.(\d{2})\.(\d{2})$/; +/** See {@link YEAR_FIRST_DOTTED_DAY}. */ +const YEAR_FIRST_COMPACT_DAY = /^(\d{4})(\d{2})(\d{2})$/; + +/** + * Read a trimmed cell as a {@link YEAR_FIRST_DOTTED_DAY} or (a text cell only) + * {@link YEAR_FIRST_COMPACT_DAY} day, or `undefined`. The day is the ISO + * `YYYY-MM-DD` spelled from the cell's own digits. + */ +function readYearFirstDay(s: string, fromText: boolean): IsoTemporalCell | undefined { + const m = YEAR_FIRST_DOTTED_DAY.exec(s) ?? (fromText ? YEAR_FIRST_COMPACT_DAY.exec(s) : null); + if (!m) return undefined; + const [, y, mo, d] = m; + if (!namesRealCalendarDay(Number(y), Number(mo), Number(d))) return undefined; + return { day: `${y}-${mo}-${d}` }; +} + /** The `HH:MM:SS` UTC clock of an instant. */ function utcClock(t: Date): string { return `${pad2(t.getUTCHours())}:${pad2(t.getUTCMinutes())}:${pad2(t.getUTCSeconds())}`; @@ -472,7 +511,9 @@ function utcClock(t: Date): string { * A text cell is read only in an {@link ISO_TEMPORAL_CELL} shape — ISO 8601, * or the export's own `YYYY-MM-DD HH:mm:ss` — or as a year-first date * ({@link YEAR_FIRST_CELL}: `2026/7/15`, `2026/7/15 9:00`, stored padded, the - * clock read exactly as the export shape's), and only on a calendar day that + * clock read exactly as the export shape's), or — a `date` / `datetime` only — + * as a dotted or compact year-first day (`2011.07.01`, `20090615`; + * {@link YEAR_FIRST_DOTTED_DAY}, #25582), and only on a calendar day that * exists. A `time` cell is read by core's one `time` rule instead * ({@link readTimeOfDayCell}, #20722): a bare `HH:MM[:SS[.f…]]` or an ISO * 8601 instant, as the write door reads it, and otherwise in the year-first @@ -561,7 +602,10 @@ export function parseDateCell( // [#20722] Core's rule is the whole verdict on an ISO 8601 `time` cell, so a // `time` cell it refuses is read in the year-first form alone — the one // reading the import has beyond the write door (#20534, maintainer ruling). - const cell = (kind === 'time' ? undefined : readIsoTemporalCell(s)) ?? readYearFirstCell(s); + // [#25582] The dotted and compact year-first days name a day and no clock, + // so a `time` cell never reads them. + const cell = (kind === 'time' ? undefined : readIsoTemporalCell(s) ?? readYearFirstDay(s, typeof raw === 'string')) + ?? readYearFirstCell(s); if (!cell) return undefined; if (cell.wall) { @@ -608,8 +652,19 @@ function outsideYears(raw: unknown, kind: 'date' | 'datetime'): { firstYear: str /** * Match a cell against a field's options, accepting **either** the option value - * (code) or its human label (case-insensitive). Returns the canonical option - * value to store, or `undefined` on no match. + * (code) or its human label, each case-insensitively. Returns the canonical + * option value to store — the declared value, never the cell's spelling — or + * `undefined` on no match. + * + * Three passes, in order: the exact value, then the label case-insensitively, + * then [#25582, maintainer ruling 6377844879 letter A] the value + * case-insensitively (`CN` stores `cn`). The value pass comes last so that a + * cell matched before it existed stores what it stored. A cell that names, by + * case alone, two declared values that differ only by case (`cn` and `CN`) is + * not resolved by that pass — picking one would be a guess — so it gets the + * no-match answer it got before. The spec cannot declare such a pair (an option + * value is a lowercase `SystemIdentifierSchema`); a programmatically registered + * object can. */ export function matchOption( raw: unknown, @@ -626,7 +681,14 @@ export function matchOption( for (const o of options) { if (o && typeof o.label === 'string' && o.label.trim().toLowerCase() === lower) return o.value; } - return undefined; + // [#25582] Case-insensitive value match, unless it names two declared values. + let hit: { value: unknown } | undefined; + for (const o of options) { + if (!o || o.value === undefined || String(o.value).toLowerCase() !== lower) continue; + if (hit && String(hit.value) !== String(o.value)) return undefined; + hit ??= { value: o.value }; + } + return hit?.value; } /** Split a multi-value cell on commas / semicolons / Chinese comma / newlines. */ diff --git a/packages/rest/src/import-year-first-day-option-case.test.ts b/packages/rest/src/import-year-first-day-option-case.test.ts new file mode 100644 index 00000000000..9c178745fc3 --- /dev/null +++ b/packages/rest/src/import-year-first-day-option-case.test.ts @@ -0,0 +1,130 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#25582, maintainer ruling 6377844879 letter A, proposals 1 and 2] + * `POST /api/v1/data/:object/import` reads two more year-first spellings of a + * `date` cell — the dotted `YYYY.MM.DD` (`2011.07.01`) and the compact + * `YYYYMMDD` (`20090615`) — and matches a select cell against the option + * VALUES case-insensitively (`CN` stores the declared `cn`), as it already + * matched the labels. Through the real route over a real `SqlDriver` + * (better-sqlite3 `:memory:`). + * + * Measured through this route by ablating each half on the built + * `@objectstack/core` the route reads: with either half removed, the ledger row + * below fails whole (`ok: 0`, `errors: 1`) on the CSV import and on the JSON + * dry run alike, as it did at the base (`ce18e73483`). At head it imports. + * + * The reader's own case table is `import-coerce.test.ts`'s `[#25582]` blocks + * in `@objectstack/core`; this file pins the door. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; +import { SysMetadataAuditObject, SysMetadataCommitObject, SysMetadataHistoryObject, SysMetadataObject } from '@objectstack/metadata-core'; +import { RestServer } from './rest-server'; + +const OBJECT = 'import_supplier_25582'; + +const SUPPLIER = { + name: OBJECT, label: 'Supplier 25582', systemFields: false, + fields: { + id: { name: 'id', type: 'text' as const, primaryKey: true, label: 'ID' }, + opened: { name: 'opened', type: 'date' as const, label: 'Opened' }, + founded: { name: 'founded', type: 'date' as const, label: 'Founded' }, + country: { + name: 'country', type: 'select' as const, label: 'Country', + options: [{ label: '中国', value: 'cn' }, { label: '美国', value: 'us' }], + }, + }, +}; + +function createMockServer() { + const noop = () => {}; + return { get: noop, post: noop, put: noop, delete: noop, patch: noop, use: noop, listen: async () => {}, close: async () => {} }; +} + +function makeRes() { + const res: any = { + write: () => true, + end: () => {}, + header: () => res, + status: (code: number) => { res._status = code; return res; }, + json: (body: any) => { res._json = body; return res; }, + }; + return res; +} + +const liveEngines: ObjectQL[] = []; +afterEach(async () => { + while (liveEngines.length) { + try { await liveEngines.pop()?.destroy(); } catch { /* noop */ } + } +}); + +async function boot() { + const engine = new ObjectQL(); + liveEngines.push(engine); + engine.registerDriver(new SqlDriver({ + client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true, + }), true); + await engine.init(); + engine.registry.registerObject(SUPPLIER as any); + await engine.syncSchemas(); + // The protocol reads the stored-metadata family; register it as a boot does. + for (const o of [SysMetadataObject, SysMetadataHistoryObject, SysMetadataAuditObject, SysMetadataCommitObject]) { + if (!engine.registry.getObject(o.name)) engine.registry.registerObject(o as any); + } + const protocol = new ObjectStackProtocolImplementation(engine as any); + const rest = new RestServer(createMockServer() as any, protocol as any, { api: { requireAuth: false } } as any); + (rest as any).resolveExecCtx = async () => ({ userId: 'test-user' }); + rest.registerRoutes(); + const importRoute = rest.getRoutes().find((r: any) => r.method === 'POST' && r.path === '/api/v1/data/:object/import') as any; + expect(importRoute).toBeDefined(); + const importRows = async (body: Record) => { + const res = makeRes(); + await importRoute.handler({ params: { object: OBJECT }, body } as any, res); + return res; + }; + return { engine, importRows }; +} + +const MAPPING = { ID: 'id', Opened: 'opened', Founded: 'founded', Country: 'country' }; + +describe('[#25582] /import reads the dotted and compact year-first day and an option value in any case', () => { + let ctx: Awaited>; + beforeEach(async () => { ctx = await boot(); }); + + it('imports the ledger row `2011.07.01`, `20090615`, `CN` (CSV) and stores the ISO days and the declared value', async () => { + const csv = ['ID,Opened,Founded,Country', 's1,2011.07.01,20090615,CN'].join('\n'); + const res = await ctx.importRows({ format: 'csv', csv, writeMode: 'insert', mapping: MAPPING }); + + expect(res._status ?? 200).toBe(200); + expect(res._json).toMatchObject({ total: 1, ok: 1, errors: 0, created: 1 }); + const row = await ctx.engine.findOne(OBJECT, { where: { id: 's1' } }); + expect({ opened: row?.opened, founded: row?.founded, country: row?.country }) + .toEqual({ opened: '2011-07-01', founded: '2009-06-15', country: 'cn' }); + }); + + it('dry-runs the same row clean (JSON) and persists nothing', async () => { + const res = await ctx.importRows({ + format: 'json', writeMode: 'insert', dryRun: true, + rows: [{ id: 'd1', opened: '2011.07.01', founded: '20090615', country: 'CN' }], + }); + expect(res._json).toMatchObject({ dryRun: true, total: 1, ok: 1, errors: 0 }); + expect(await ctx.engine.findOne(OBJECT, { where: { id: 'd1' } })).toBeNull(); + }); + + it('still refuses a compact day that does not exist, and an unknown option, as that row\'s error', async () => { + const res = await ctx.importRows({ + format: 'json', writeMode: 'insert', + rows: [{ id: 'bad', founded: '20090230' }, { id: 'odd', country: 'DE' }, { id: 'good', founded: '20090615' }], + }); + expect(res._json).toMatchObject({ total: 3, ok: 1, errors: 2, created: 1 }); + expect(res._json.results[0]).toMatchObject({ row: 1, ok: false, action: 'failed', field: 'founded', code: 'invalid_date' }); + expect(res._json.results[1]).toMatchObject({ row: 2, ok: false, action: 'failed', field: 'country', code: 'invalid_option' }); + expect(await ctx.engine.findOne(OBJECT, { where: { id: 'bad' } })).toBeNull(); + expect((await ctx.engine.findOne(OBJECT, { where: { id: 'good' } }))?.founded).toBe('2009-06-15'); + }); +});