diff --git a/apps/site/public/llms-small.txt b/apps/site/public/llms-small.txt index 3833bb1..650dd0e 100644 --- a/apps/site/public/llms-small.txt +++ b/apps/site/public/llms-small.txt @@ -49,12 +49,19 @@ Distinct from success `alternatives`, failure `candidate`, and issue `suggestion import { createLingo } from "@pascal-app/lingo" import { es } from "@pascal-app/lingo/locales/es" import { fr } from "@pascal-app/lingo/locales/fr" +import { ja } from "@pascal-app/lingo/locales/ja" import { zh } from "@pascal-app/lingo/locales/zh" -const lingo = createLingo({ locales: [es, fr, zh] }) +const lingo = createLingo({ locales: [es, fr, ja, zh] }) lingo.parseQuantity("dos kg") // auto-detected as es; result.locale === "es" lingo.parseRange("entre 5 et 10 kg", { locale: "fr" }) lingo.parseQuantity("5公斤", { locale: "zh" }) +lingo.parseQuantity("mille cinq cents metres", { locale: "fr" }) // 1500 m + +// Comparators that FOLLOW the quantity (CJK); these return ranges, not quantities. +lingo.parseRange("5キロ未満", { locale: "ja" }) // { min: null, max: 5 kg } +lingo.parseRange("5公斤以上", { locale: "zh" }) // { min: 5 kg, max: null } +lingo.parseQuantity("100元", { locale: "zh" }) // 100 CNY (pack currency default) ``` Packs: `en`, `en-gb`, `es`, `fr`, `pt`, `zh`, `ja`. Omit `locale` to detect among loaded packs plus English; pass `locale` for known fields. English is built in and does not require a pack. `parseDate(text, { locale, localePacks, now })` can use loaded packs for relative date vocabulary. @@ -62,11 +69,13 @@ Packs: `en`, `en-gb`, `es`, `fr`, `pt`, `zh`, `ja`. Omit `locale` to detect amon Pack notes: - `en`: built-in English quantity/date grammar; wave-1 adds spoken decimals and date idioms such as `two point five kg` and `quarter of five`. - `en-gb`: English plus day-first numeric dates and UK weekday offsets (`Monday week`, `Tuesday fortnight`). -- `es`: Spanish number/range words, Romance composition, spoken decimals, approximants, and date clock/edge idioms. -- `fr`: French number/range words, exhaustive CLDR-style vigesimal compounds, spoken decimals, approximants, and weekday-offset/clock idioms. -- `pt`: Portuguese number/range words, compound hundreds/tens, spoken decimals, approximants, and clock/day-offset idioms. -- `zh`: Chinese CJK number walker, scale grouping, post-unit half, duration counters, period edges, and day offsets. CJK spoken clock (`下午3点半`) is deferred. -- `ja`: Japanese CJK number walker, scale grouping, post-unit half, duration counters, wave-dash ranges, and double-step period modifiers. CJK spoken clock (`午後3時半`) is deferred. +- `es`: Spanish number/range words, Romance composition, spoken decimals, approximants, and date clock/edge idioms. Long scale: `mil millones` = 1e9, `billón` = 1e12. Ordinal days (`el 1º de marzo`). +- `fr`: French number/range words, exhaustive CLDR-style vigesimal compounds, spoken decimals, approximants, and weekday-offset/clock idioms. Long scale: `milliard` = 1e9, `billion` = 1e12. Ordinal days (`le 1er mars`). +- `pt`: Portuguese number/range words, compound hundreds/tens (`cento e vinte`, `mil e quinhentos`), spoken decimals, approximants, clock/day-offset idioms, `-feira` weekdays, and locative date fillers (`na proxima segunda-feira`). +- `zh`: Chinese CJK number walker, scale grouping, post-unit half, duration counters, period edges, day offsets, postpositional bounds (`5公斤以上`, `5公斤以下`, `5公斤以内`), CNY default plus `元`/`块` aliases, suffix dates (`2026年3月5日`, `3月5日`), weekdays (`星期三`, `周三`), and spoken clocks with day periods (`下午3点半`, `三点一刻`, `明天下午3点`). +- `ja`: Japanese CJK number walker, scale grouping, post-unit half, duration counters, wave-dash ranges, double-step period modifiers, postpositional bounds (`5キロ未満`, `5キロ以上`, `5キロ超`), JPY default plus `円` aliases, suffix dates (`2026年3月5日`), weekdays (`水曜日`), and spoken clocks with day periods (`午後3時半`, `明日午後3時`). + +Note: `以上`/`未満`-style comparators produce an open RANGE, so read them with `parseRange`/`rangeField`, not `parseQuantity`. The shared `¥`/`¥` symbol resolves per locale (CNY under `zh`, JPY under `ja`). Verified locale examples (fixed `now = 2026-07-03 14:30` local time for date examples): diff --git a/apps/site/src/components/site/parse-playground.tsx b/apps/site/src/components/site/parse-playground.tsx index b5e64aa..f1657ae 100644 --- a/apps/site/src/components/site/parse-playground.tsx +++ b/apps/site/src/components/site/parse-playground.tsx @@ -24,12 +24,31 @@ const localeLingo = createLingo({ locales: [es, fr, pt, zh, ja, enGb] }) const EXAMPLES: Record = { auto: ['72 in to cm', 'dos kg', 'entre 5 et 10 kg', '5公斤', '暑い'], - en: ['2 ft', '72 in to cm', 'between 5 and 10 kg', '5 meters', "it's hot"], - es: ['dos kg', 'entre 5 y 10 kg', '72 pulgadas a cm', 'dos metros y medio', 'al menos 2 m'], - fr: ['deux kg', 'entre 5 et 10 kg', '72 pouces en cm', 'deux metres et demi', 'au moins 2 m'], - pt: ['dois kg', 'entre 5 e 10 kg', '72 polegadas em cm', 'pelo menos 2 m'], - zh: ['5公斤', '3米', '5到10公斤', '很热'], - ja: ['5キロ', '3メートル', '5から10キロ', '暑い'], + en: [ + '2 ft', + '72 in to cm', + 'between 5 and 10 kg', + 'between one thousand and two thousand meters', + "it's hot", + ], + es: [ + 'dos kg', + 'entre 5 y 10 kg', + '72 pulgadas a cm', + 'ciento veinte kg', + 'mil millones de kg', + 'al menos 2 m', + ], + fr: [ + 'deux kg', + 'entre 5 et 10 kg', + '72 pouces en cm', + 'mille cinq cents metres', + 'deux metres et demi', + ], + pt: ['dois kg', 'entre 5 e 10 kg', 'cento e vinte metros', 'mil e quinhentos metros'], + zh: ['5公斤', '5公斤以上', '5公斤以下', '五公斤左右', '100元', '很热'], + ja: ['5キロ', '5キロ未満', '5キロ以上', '5キロぐらい', '1000円', '暑い'], 'en-gb': ['12 stone', '3 quid', 'roundabout 2 m'], } diff --git a/apps/site/src/lib/code-snippets.ts b/apps/site/src/lib/code-snippets.ts index b546c99..d21baf4 100644 --- a/apps/site/src/lib/code-snippets.ts +++ b/apps/site/src/lib/code-snippets.ts @@ -164,13 +164,19 @@ humanizeDuration(5400, { style: "natural" }) // "an hour and a half"` export const localeSnippet = `import { createLingo } from "@pascal-app/lingo" import { es } from "@pascal-app/lingo/locales/es" import { fr } from "@pascal-app/lingo/locales/fr" +import { ja } from "@pascal-app/lingo/locales/ja" import { zh } from "@pascal-app/lingo/locales/zh" -const lingo = createLingo({ locales: [es, fr, zh] }) +const lingo = createLingo({ locales: [es, fr, ja, zh] }) lingo.parseQuantity("dos kg") // auto-detected as es lingo.parseRange("entre 5 et 10 kg", { locale: "fr" }) lingo.parseQuantity("5公斤", { locale: "zh" }) +lingo.parseQuantity("mille cinq cents metres", { locale: "fr" }) // 1500 m + +// Comparators that follow the quantity, as CJK writes them. +lingo.parseRange("5キロ未満", { locale: "ja" }) // < 5 kg +lingo.parseRange("5公斤以上", { locale: "zh" }) // >= 5 kg // Success results expose the resolved profile. lingo.parse("dos kg").locale // "es"` diff --git a/apps/site/src/lib/docs.md.ts b/apps/site/src/lib/docs.md.ts index f26bba8..8bfd800 100644 --- a/apps/site/src/lib/docs.md.ts +++ b/apps/site/src/lib/docs.md.ts @@ -318,7 +318,9 @@ lingo.parse('72 in to cm') // locale: 'en'`, '', 'Published packs: `@pascal-app/lingo/locales/en`, `en-gb`, `es`, `fr`, `pt`, `zh`, and `ja`. Pack-owned CJK unit aliases and fuzzy words are only registered when their pack is loaded. Passing an explicit locale that was not loaded returns `LOCALE_NOT_LOADED` instead of silently falling back to English.', '', - 'Date parsing accepts the same packs directly: `parseDate("mañana", { now, locale: "es", localePacks: [es] })`.', + 'Each pack reads its language the way people write it, not a transliteration of English. Romance packs cover tens/hundreds/scale words (`ciento veinte`, `mille cinq cents`, `mil millones`), ordinal days (`le 1er mars`), and spoken decimals (`dos coma cinco`). The Chinese and Japanese packs cover grouped numerals (`一百五十万`), comparators that follow the quantity (`5キロ未満`, `5公斤以上`), trailing approximants (`五公斤左右`), and per-locale currency defaults — the shared `¥` resolves to CNY under `zh` and JPY under `ja`.', + '', + 'Date parsing accepts the same packs directly: `parseDate("mañana", { now, locale: "es", localePacks: [es] })`. Localized date grammar includes spoken clocks (`las tres menos cuarto`, `deux heures et quart`), period edges (`a finales de mes`), weekday offsets (`lundi en huit`), and CJK calendar forms — suffix dates (`2026年3月5日`, `3月5日`), weekdays (`星期三`, `水曜日`), clocks with day periods (`午後3時半`, `下午3点`), and date+time written without a space (`明天下午3点`).', '', '## Catalog', '', diff --git a/packages/lingo/CHANGELOG.md b/packages/lingo/CHANGELOG.md index 4ea9a74..8661338 100644 --- a/packages/lingo/CHANGELOG.md +++ b/packages/lingo/CHANGELOG.md @@ -7,8 +7,56 @@ change**, even if the API is untouched. ## [Unreleased] +### Added + +- Chinese and Japanese calendar and clock grammar: numeric dates written with + suffixes (`2026年3月5日`, `3月5日`, and years spelled digit-by-digit as + `二〇二六年`), weekdays (`星期三`, `周三`, `水曜日`), clocks closed by + `点`/`時`/`分`/`秒` with day periods (`下午3点`, `午前9時半`, `3点一刻`), and + date+time compounds written with no separating space (`明天下午3点`, + `明日午後3時`). +- Postpositional bound phrases for scripts that put the comparator after the + quantity: `5キロ未満`, `5キロ以上`, `5公斤以下`, `5公斤以内`, `5キロ超`. + Previously `5キロ未満` parsed as a bare `5 kg`, dropping the bound. +- Chinese and Japanese default currencies, so the shared `¥`/`¥` symbol + resolves per locale (`¥100` is CNY under `zh`, JPY under `ja`), plus the + `元`/`圆`/`块`/`人民币` and `円`/`えん` aliases. +- French long-scale number words (`milliard` = 10^9, `billion` = 10^12), + Spanish `mil millones`/`billón`, Portuguese `cento`, and Romance ordinal + day-of-month forms (`le 1er mars`, `el 1º de marzo`). +- A per-locale benchmark suite (`bun run bench`) so multi-language throughput is + measured rather than assumed. +- Per-locale corpus contracts gained 96 rows and the English contract 5, + covering the grammar above plus the number-word fixes below. + +### Fixed + +- Number words: hundreds now multiply only the 1..99 group in front of them, so + French `mille cinq cents` is 1500 (was 100500). A banked smaller scale + multiplies the next one, so Spanish `mil millones` is 10^9 (was 1,001,000) and + `dos mil millones` is 2×10^9. Both were silent wrong answers. +- `between A and B` now keeps the and-word for the range when both sides are + spelled scale words: `between one thousand and two thousand meters`, + `entre mille et deux mille`, `entre mil y dos mil` (all previously failed to + parse). `between five and a half and ten kg` still reads 5.5..10 — the + fraction tail binds tighter than the range separator. +- Portuguese `cento e vinte` (120) and `mil e quinhentos` (1500) parse; the + and-word after a bare scale word links it to its remainder. +- Portuguese locative contractions are date fillers, so + `na proxima segunda-feira` resolves. +- Scripts written without word spaces no longer swallow grammar words glued to a + quantity (`三至五天`, `5公斤左右`). Unit aliases stay atomic through the split, + so `一時間半` remains 1.5 hours rather than splitting at the range word `間`. + ### Changed +- Resolved language profiles and locale-detection scans are memoized per pack + set, so repeated parses with `locale`/auto-detection no longer re-merge packs + or re-tokenize for detection on every call. +- Size budgets recalibrated once for this wave (see D70 in `wiki/decisions.md` + for the measured numbers and rationale); `scripts/size.mjs` remains the source + of truth. No previously-pinned corpus interpretation changed — the wave is + additive. - README (npm): repo-relative links (`docs/recipes.md`, `plans/`, `wiki/`) now point at absolute GitHub URLs so they resolve on npmjs.com, and the docs site plus the agent `llms.txt` index are linked from the top. diff --git a/packages/lingo/README.md b/packages/lingo/README.md index a81731f..92e8d19 100644 --- a/packages/lingo/README.md +++ b/packages/lingo/README.md @@ -559,6 +559,11 @@ spanish.ok && spanish.quantity.to('kg').value // 35 const chinese = app.parseQuantity('三十五公斤', { locale: 'zh' }) chinese.ok && chinese.quantity.to('kg').value // 35 +// comparators that follow the quantity, as CJK writes them +app.parse('5キロ未満', { locale: 'ja' }) // < 5 kg (open range) +app.parse('5公斤以上', { locale: 'zh' }) // ≥ 5 kg +app.parse('100元', { locale: 'zh' }) // 100 CNY + const clock = parseDate('las tres menos cuarto', { locale: 'es', localePacks: [es], @@ -566,17 +571,22 @@ const clock = parseDate('las tres menos cuarto', { }) clock.ok && [clock.date.getHours(), clock.date.getMinutes()] // [2, 45] +const cjkDate = parseDate('2026年3月5日', { locale: 'zh', localePacks: [zh] }) +cjkDate.ok && [cjkDate.date.getMonth() + 1, cjkDate.date.getDate()] // [3, 5] (local) + // omit `locale` to auto-detect among the loaded packs ``` Packs (`en`, `en-gb`, `es`, `fr`, `pt`, `zh`, `ja`) are additive and tree-shakeable; they add number words, units, ranges, and relative dates (through -`@pascal-app/lingo/date`) for their language. The first idiom wave includes -Romance tens/hundreds/decimal words, CJK scale/grouped numbers and post-unit -half, localized clock phrases, period edges, weekday offsets, and locale unit -words in date/duration parsing. Requesting an unloaded locale returns -`LOCALE_NOT_LOADED` rather than silently parsing as English; `humanizeDate()` output -stays English for now. +`@pascal-app/lingo/date`) for their language. Coverage includes Romance +tens/hundreds/scale words (`ciento veinte`, `mille cinq cents`, `mil millones`), +CJK scale/grouped numbers and post-unit half, postpositional comparators +(`5キロ未満`, `5公斤以上`), per-locale currency defaults (`¥` is CNY under `zh`, +JPY under `ja`), localized clock phrases and period edges, and CJK calendar +dates and clocks (`2026年3月5日`, `午後3時半`, `明天下午3点`). Requesting an +unloaded locale returns `LOCALE_NOT_LOADED` rather than silently parsing as +English; `humanizeDate()` output stays English for now. The global `lingo()` is itself a `createLingo()` singleton. One code path, two tiers of convenience. diff --git a/packages/lingo/llms.txt b/packages/lingo/llms.txt index 3833bb1..650dd0e 100644 --- a/packages/lingo/llms.txt +++ b/packages/lingo/llms.txt @@ -49,12 +49,19 @@ Distinct from success `alternatives`, failure `candidate`, and issue `suggestion import { createLingo } from "@pascal-app/lingo" import { es } from "@pascal-app/lingo/locales/es" import { fr } from "@pascal-app/lingo/locales/fr" +import { ja } from "@pascal-app/lingo/locales/ja" import { zh } from "@pascal-app/lingo/locales/zh" -const lingo = createLingo({ locales: [es, fr, zh] }) +const lingo = createLingo({ locales: [es, fr, ja, zh] }) lingo.parseQuantity("dos kg") // auto-detected as es; result.locale === "es" lingo.parseRange("entre 5 et 10 kg", { locale: "fr" }) lingo.parseQuantity("5公斤", { locale: "zh" }) +lingo.parseQuantity("mille cinq cents metres", { locale: "fr" }) // 1500 m + +// Comparators that FOLLOW the quantity (CJK); these return ranges, not quantities. +lingo.parseRange("5キロ未満", { locale: "ja" }) // { min: null, max: 5 kg } +lingo.parseRange("5公斤以上", { locale: "zh" }) // { min: 5 kg, max: null } +lingo.parseQuantity("100元", { locale: "zh" }) // 100 CNY (pack currency default) ``` Packs: `en`, `en-gb`, `es`, `fr`, `pt`, `zh`, `ja`. Omit `locale` to detect among loaded packs plus English; pass `locale` for known fields. English is built in and does not require a pack. `parseDate(text, { locale, localePacks, now })` can use loaded packs for relative date vocabulary. @@ -62,11 +69,13 @@ Packs: `en`, `en-gb`, `es`, `fr`, `pt`, `zh`, `ja`. Omit `locale` to detect amon Pack notes: - `en`: built-in English quantity/date grammar; wave-1 adds spoken decimals and date idioms such as `two point five kg` and `quarter of five`. - `en-gb`: English plus day-first numeric dates and UK weekday offsets (`Monday week`, `Tuesday fortnight`). -- `es`: Spanish number/range words, Romance composition, spoken decimals, approximants, and date clock/edge idioms. -- `fr`: French number/range words, exhaustive CLDR-style vigesimal compounds, spoken decimals, approximants, and weekday-offset/clock idioms. -- `pt`: Portuguese number/range words, compound hundreds/tens, spoken decimals, approximants, and clock/day-offset idioms. -- `zh`: Chinese CJK number walker, scale grouping, post-unit half, duration counters, period edges, and day offsets. CJK spoken clock (`下午3点半`) is deferred. -- `ja`: Japanese CJK number walker, scale grouping, post-unit half, duration counters, wave-dash ranges, and double-step period modifiers. CJK spoken clock (`午後3時半`) is deferred. +- `es`: Spanish number/range words, Romance composition, spoken decimals, approximants, and date clock/edge idioms. Long scale: `mil millones` = 1e9, `billón` = 1e12. Ordinal days (`el 1º de marzo`). +- `fr`: French number/range words, exhaustive CLDR-style vigesimal compounds, spoken decimals, approximants, and weekday-offset/clock idioms. Long scale: `milliard` = 1e9, `billion` = 1e12. Ordinal days (`le 1er mars`). +- `pt`: Portuguese number/range words, compound hundreds/tens (`cento e vinte`, `mil e quinhentos`), spoken decimals, approximants, clock/day-offset idioms, `-feira` weekdays, and locative date fillers (`na proxima segunda-feira`). +- `zh`: Chinese CJK number walker, scale grouping, post-unit half, duration counters, period edges, day offsets, postpositional bounds (`5公斤以上`, `5公斤以下`, `5公斤以内`), CNY default plus `元`/`块` aliases, suffix dates (`2026年3月5日`, `3月5日`), weekdays (`星期三`, `周三`), and spoken clocks with day periods (`下午3点半`, `三点一刻`, `明天下午3点`). +- `ja`: Japanese CJK number walker, scale grouping, post-unit half, duration counters, wave-dash ranges, double-step period modifiers, postpositional bounds (`5キロ未満`, `5キロ以上`, `5キロ超`), JPY default plus `円` aliases, suffix dates (`2026年3月5日`), weekdays (`水曜日`), and spoken clocks with day periods (`午後3時半`, `明日午後3時`). + +Note: `以上`/`未満`-style comparators produce an open RANGE, so read them with `parseRange`/`rangeField`, not `parseQuantity`. The shared `¥`/`¥` symbol resolves per locale (CNY under `zh`, JPY under `ja`). Verified locale examples (fixed `now = 2026-07-03 14:30` local time for date examples): diff --git a/packages/lingo/scripts/bench.mjs b/packages/lingo/scripts/bench.mjs index 1b7bbf1..125b4d6 100644 --- a/packages/lingo/scripts/bench.mjs +++ b/packages/lingo/scripts/bench.mjs @@ -13,6 +13,37 @@ const DIST_DOM = join(ROOT, 'dist/dom/index.js') const DEFAULT_BASELINE = join(ROOT, 'bench/baseline-node.json') const NOW = new Date(2026, 6, 3, 14, 30, 0) +/** + * Locale corpora stay hand-written: the point is to measure the profile + * resolution and auto-detection paths on realistic native input, not to + * re-measure the English grammar through a translated alias table. + */ +const LOCALE_CORPORA = { + cjk: [ + '5公斤', + '三十五公斤', + '一百五十米', + '5公斤左右', + '3万5千公斤', + '5キロ', + '三十五キロ', + '約5キロ', + ], + english: ['5 kg', '72 in to cm', 'between 5 and 10 kg', '5\'11"', 'two hundred grams'], + romance: [ + 'cinq kg', + 'deux virgule cinq kg', + 'quatre-vingt-dix kg', + 'environ 5 kilos', + 'entre 5 et 10 kg', + 'treinta y cinco kg', + 'mas o menos 5 kilos', + 'quinientos gramos', + 'dois virgula cinco kg', + 'mais ou menos 5 quilos', + ], +} + const args = parseArgs(process.argv.slice(2)) let sink = 0 @@ -565,6 +596,7 @@ async function runBackend() { const main = await import(`../dist/index.js?bench=${Date.now()}`) const date = await import(`../dist/date/index.js?bench=${Date.now()}`) const complete = await import(`../dist/complete/index.js?bench=${Date.now()}`) + const localePacks = await loadLocalePacks() const corpora = buildBackendCorpora(main.allKinds) const lengthMeters = { kind: 'length', unit: 'm' } const massKind = { kind: 'mass' } @@ -663,6 +695,7 @@ async function runBackend() { samples: corpora.freeText, fn: (input) => main.findQuantities(input), }, + ...localeSuites(main, localePacks), ] const probes = [ @@ -713,6 +746,73 @@ async function runBackend() { return report } +async function loadLocalePacks() { + const ids = ['es', 'fr', 'pt', 'zh', 'ja'] + const stamp = Date.now() + const packs = {} + for (const id of ids) { + packs[id] = (await import(`../dist/locales/${id}.js?bench=${stamp}`)).default + } + return packs +} + +/** + * Locale suites exist to keep the multi-language paths honest: a resolved + * profile is ~40 merged tables, and auto-detection scores every loaded pack, so + * both are cached. These suites fail loudly (as a throughput drop) if a change + * reintroduces per-parse profile merging or per-pack re-tokenization. + */ +function localeSuites(main, packs) { + const all = [packs.es, packs.fr, packs.pt, packs.zh, packs.ja] + const single = main.createLingo({ locales: [packs.fr] }) + const loaded = main.createLingo({ locales: all }) + return [ + { + name: 'locale explicit (1 pack)', + group: 'backend/locale', + samples: LOCALE_CORPORA.romance.slice(0, 4), + fn: (input) => single.parse(input, { locale: 'fr' }), + }, + { + name: 'locale explicit (5 packs)', + group: 'backend/locale', + samples: LOCALE_CORPORA.romance, + fn: (input) => loaded.parse(input, { locale: localeFor(input) }), + }, + { + name: 'locale english w/ packs loaded', + group: 'backend/locale', + samples: LOCALE_CORPORA.english, + fn: (input) => loaded.parse(input, { locale: 'en' }), + }, + { + name: 'locale auto-detect romance', + group: 'backend/locale', + samples: LOCALE_CORPORA.romance, + fn: (input) => loaded.parse(input), + }, + { + name: 'locale auto-detect cjk', + group: 'backend/locale', + samples: LOCALE_CORPORA.cjk, + fn: (input) => loaded.parse(input), + }, + { + name: 'locale auto-detect english', + group: 'backend/locale', + samples: LOCALE_CORPORA.english, + fn: (input) => loaded.parse(input), + }, + ] +} + +function localeFor(input) { + if (LOCALE_CORPORA.romance.indexOf(input) < 5) { + return 'fr' + } + return LOCALE_CORPORA.romance.indexOf(input) < 8 ? 'es' : 'pt' +} + function runSuites(suites) { return suites.map((suite) => { const iterations = suite.iterations ?? args.iterations diff --git a/packages/lingo/scripts/size.mjs b/packages/lingo/scripts/size.mjs index 8f89632..254f838 100644 --- a/packages/lingo/scripts/size.mjs +++ b/packages/lingo/scripts/size.mjs @@ -143,8 +143,12 @@ function check(label, size, budget) { // approximatePhrases) + CJK number engine (sub-token segmentation, 万/亿 // grouping, elliptical/mixed forms, wave-dash + adjacent ranges, post-unit 半). // Measured 38.19 after golfing; capability is product (D14 pattern). +// 39.4 (was 38.3): D70 — wave-2 idiom engine: profile-driven splitting of glued +// CJK grammar words, postpositional bounds (5キロ未満), and number-word +// arithmetic fixes (hundreds bind to the preceding group, scale chaining, +// `noAnd` threading). Measured 39.18. const full = await bundleStdin(`export * from './src/index.ts'`) -check('lingo (full)', full, 38_300) +check('lingo (full)', full, 39_400) if (has('src/locales/es.ts')) { const enLocale = await bundleStdin(`export * from './src/locales/en.ts'`) @@ -174,12 +178,17 @@ if (has('src/locales/es.ts')) { // 1.3 (was 1.2): D69 — plan 033: CJK number tables for the D68 walker // (两/幺/点/半, scales), duration unit aliases (个小时/時間), trailing approx // (左右/上下), day offsets (前天/一昨日), period edges, 円/JPY. + // 2.0 (was 1.3): D70 — calendar/clock vocab (weekdays, months, 年月日 numeric + // suffixes, 点/分/秒 clock, 上午/下午 day periods, 半/一刻 minute words), + // postpositional bound phrases, and the CNY default + 元/块 aliases. const zhLocale = await bundleStdin(`export * from './src/locales/zh.ts'`) - check('./locales/zh (standalone data)', zhLocale, 1300) + check('./locales/zh (standalone data)', zhLocale, 2000) // 1.4 (was 1.2): D69 — see zh note; ja adds 再来週/先々週 modifiers and // 今朝/今晩 day-part phrases. + // 2.0 (was 1.4): D70 — see zh note; ja adds 時/分 clock, 午前/午後 periods, + // 曜日 weekdays, the の filler, and 未満/以上/超 postpositional bounds. const jaLocale = await bundleStdin(`export * from './src/locales/ja.ts'`) - check('./locales/ja (standalone data)', jaLocale, 1400) + check('./locales/ja (standalone data)', jaLocale, 2000) const enGbLocale = await bundleStdin(`export * from './src/locales/en-gb.ts'`) check('./locales/en-gb (standalone data)', enGbLocale, 250) @@ -197,13 +206,16 @@ if (has('src/locales/es.ts')) { ) // 2.0 (was 1.6): D69 — plan 033 CJK idiom pack data (see zh/ja standalone // notes); low zh/ja gzip dedup because the scripts differ. - check('./locales zh+ja (marginal over full)', withCjkLocales - full, 2000) + // 3.0 (was 2.0): D70 — calendar/clock/bound vocab in both packs (see zh/ja + // standalone notes). + check('./locales zh+ja (marginal over full)', withCjkLocales - full, 3000) const withAllLocales = await bundleStdin( `export * from './src/index.ts'; export { es } from './src/locales/es.ts'; export { fr } from './src/locales/fr.ts'; export { pt } from './src/locales/pt.ts'; export { zh } from './src/locales/zh.ts'; export { ja } from './src/locales/ja.ts'; export { enGb } from './src/locales/en-gb.ts'`, ) // 6.3 (was 5.8): D69 — plan 033 romance idiom packs (see above). // (raised again for restored breadth entries — coverage beats pack-byte golf, D14) - check('./locales all loaded (marginal over full)', withAllLocales - full, 6300) + // 6.9 (was 6.3): D70 — zh/ja calendar/clock/bound vocab (see above). + check('./locales all loaded (marginal over full)', withAllLocales - full, 6900) } // 19.9 (was 19.6): D48 — shared parser recognizes GBP pence idioms and @@ -248,8 +260,11 @@ if (has('src/locales/es.ts')) { // 25.6 (was 24.2): D68 — wave-1 idiom engine mechanisms live in the shared // number/parse layer (Romance composition fields + CJK number walker), so // BYO-registry cores get them too. Measured 25.46 after golfing. +// 26.8 (was 25.6): D70 — wave-2 mechanisms in the shared layer: profile-driven +// splitting of glued grammar words (unit aliases stay atomic), postpositional +// bound phrases, and number-word arithmetic fixes. Measured 26.56. const core = await bundleStdin(`export * from './src/core/index.ts'`) -check('./core (engine, no unit data)', core, 25_600) +check('./core (engine, no unit data)', core, 26_800) if (has('src/date/index.ts')) { const dateAlone = await bundleStdin(`export * from './src/date/index.ts'`) @@ -302,7 +317,10 @@ if (has('src/date/index.ts')) { // 41.0 (was 39.7): D69 — localized date grammar seams (clock past/to/minute // words, period edges, weekday offsets, afterNext/beforeLast, day-part // compounds, localized durations). Measured 40.85 after golfing. - check('./date (standalone, incl. engine)', dateAlone, 41_000) + // 43.3 (was 41.0): D70 — suffix-delimited date/clock grammar (date/suffix.ts, + // date/numeral.ts): 年月日 numeric dates, 点/時/分/秒 clocks, day periods, + // unspaced date+time splitting, glued affix matching. Measured 43.01. + check('./date (standalone, incl. engine)', dateAlone, 43_300) const withDate = await bundleStdin( `export * from './src/index.ts'; export * from './src/date/index.ts'`, ) @@ -324,7 +342,9 @@ if (has('src/date/index.ts')) { // 13.1 (was 12.9): D66 — date-only parser readers for locale date vocab. // 14.4 (was 13.1): D69 — localized date grammar consumers live entirely in // the date module (see standalone note). Measured 14.29 after golfing. - check('./date (marginal over full)', withDate - full, 14_400) + // 15.7 (was 14.4): D70 — the suffix date/clock grammar lands entirely in the + // date module (see standalone note). Measured 15.46. + check('./date (marginal over full)', withDate - full, 15_700) } if (has('src/dom/index.ts')) { @@ -455,7 +475,9 @@ if (has('src/ai/index.ts')) { // 16.1 (was 15.7): D66 — /ai date fields bundle the locale-aware date parser. // 17.4 (was 16.1): D69 — the localized date grammar cascades through the // bundled date module (see ./date notes). Measured 17.27 after golfing. - check('./ai (marginal over full)', withAi - full, 17_400) // D30: +notation in shared renderNumber + // 18.6 (was 17.4): D70 — the suffix date/clock grammar cascades through the + // bundled date module; no /ai code changed. Measured 18.39. + check('./ai (marginal over full)', withAi - full, 18_600) // D30: +notation in shared renderNumber if (has('src/mcp/index.ts')) { const withMcp = await bundleStdin( `export * from './src/index.ts'; export * from './src/ai/index.ts'; export * from './src/mcp/index.ts'`, diff --git a/packages/lingo/src/date/absolute.ts b/packages/lingo/src/date/absolute.ts index 16dce78..4c672ca 100644 --- a/packages/lingo/src/date/absolute.ts +++ b/packages/lingo/src/date/absolute.ts @@ -2,6 +2,7 @@ import type { LingoIssue } from '../core/types' import { parseYear, sameDay, startOfDay, validDateTime } from './civil' import type { DateAlternative, DateGrain } from './parse' import { type CoreDate, core, issue, knownFor, needsNow, type P, stripDateFillers } from './state' +import { parseSuffixDate } from './suffix' import { MONTHS as EN_MONTHS } from './vocab' export function parseAbsolute(p: P, start: number, end: number): CoreDate | null { @@ -38,10 +39,13 @@ export function parseAbsolute(p: P, start: number, end: number): CoreDate | null return numericDateCore(p, numeric, start, end) } - const md = new RegExp( - `^(${monthAlternation(p)})\\s+(\\d{1,2})(?:st|nd|rd|th)?(?:,?\\s+('?\\d{2}|\\d{4}))?$`, - 'i', - ).exec(source) + const suffixed = parseSuffixDate(p, start, end) + if (suffixed) { + return suffixed + } + + const patterns = monthPatterns(p) + const md = patterns.monthDay.exec(source) if (md) { return monthDayCore( p, @@ -53,10 +57,7 @@ export function parseAbsolute(p: P, start: number, end: number): CoreDate | null end, ) } - const dm = new RegExp( - `^(\\d{1,2})(?:st|nd|rd|th)?(?:\\s+of)?\\s+(${monthAlternation(p)})(?:,?\\s+('?\\d{2}|\\d{4}))?$`, - 'i', - ).exec(source) + const dm = patterns.dayMonth.exec(source) if (dm) { return monthDayCore( p, @@ -68,7 +69,7 @@ export function parseAbsolute(p: P, start: number, end: number): CoreDate | null end, ) } - const my = new RegExp(`^(${monthAlternation(p)})\\s+('?\\d{2}|\\d{4})$`, 'i').exec(source) + const my = patterns.monthYear.exec(source) if (my) { const year = parseYear(my[2]) if (year === undefined) { @@ -141,10 +142,50 @@ export function monthDayCore( return year === undefined ? needsNow(parsed) : parsed } -function monthAlternation(p: P): string { - return Object.keys(dateMonths(p)) +interface MonthPatterns { + dayMonth: RegExp + monthDay: RegExp + monthYear: RegExp +} + +const EN_ORDINAL_SUFFIXES: readonly string[] = ['st', 'nd', 'rd', 'th'] +const patternCache = new WeakMap, MonthPatterns>() + +/** + * Month alternations are derived from the profile's month table, so they are + * compiled once per table rather than rebuilt (sort + join + three `RegExp` + * constructions) on every date parse. + */ +function monthPatterns(p: P): MonthPatterns { + const months = dateMonths(p) + const cached = patternCache.get(months) + if (cached) { + return cached + } + const alternation = Object.keys(months) .sort((a, b) => b.length - a.length) .join('|') + const ord = ordinalSuffixPattern(p) + const year = "(?:,?\\s+('?\\d{2}|\\d{4}))?" + const patterns: MonthPatterns = { + dayMonth: new RegExp(`^(\\d{1,2})${ord}(?:\\s+of)?\\s+(${alternation})${year}$`, 'i'), + monthDay: new RegExp(`^(${alternation})\\s+(\\d{1,2})${ord}${year}$`, 'i'), + monthYear: new RegExp(`^(${alternation})\\s+('?\\d{2}|\\d{4})$`, 'i'), + } + patternCache.set(months, patterns) + return patterns +} + +/** + * Ordinal markers allowed between a day number and its month. Packs that do not + * declare their own keep the English suffixes, which are inert in other + * languages because the marker is optional. + */ +function ordinalSuffixPattern(p: P): string { + const suffixes = p.profile.date?.ordinalSuffixes?.length + ? p.profile.date.ordinalSuffixes + : EN_ORDINAL_SUFFIXES + return `(?:${suffixes.map((s) => s.replace(/[^\w]/g, '\\$&')).join('|')})?` } function dateMonths(p: P): Record { diff --git a/packages/lingo/src/date/numeral.ts b/packages/lingo/src/date/numeral.ts new file mode 100644 index 0000000..508b553 --- /dev/null +++ b/packages/lingo/src/date/numeral.ts @@ -0,0 +1,78 @@ +import type { P } from './state' + +/** + * Read a bare number written with the pack's `numerals` table — Arabic digits + * (`15`) or the locale's own digit characters composed positionally around the + * ten mark (`十五`, `二十`, `二十五`). Returns null when the text is not a + * number in this locale, so callers can fall through to other grammar. + */ +export function readLocaleNumber(p: P, text: string): number | null { + if (/^\d+$/.test(text)) { + return Number(text) + } + const numerals = p.profile.numerals + if (!numerals) { + return null + } + if (text === '十') { + return 10 + } + const ten = text.indexOf('十') + if (ten >= 0) { + const left = ten === 0 ? 1 : (numerals[text.slice(0, ten)] ?? Number.NaN) + const right = ten === text.length - 1 ? 0 : (numerals[text.slice(ten + 1)] ?? Number.NaN) + const value = left * 10 + right + return Number.isFinite(value) ? value : null + } + const value = numerals[text] + if (value !== undefined) { + return value + } + // Years are spelled out digit by digit rather than positionally: + // 二〇二六年 is 2026, not "two, zero, two, six" summed. + return readDigitRun(text, numerals) +} + +function readDigitRun(text: string, numerals: Record): number | null { + if ([...text].length < 2) { + return null + } + let out = '' + for (const ch of text) { + const digit = numerals[ch] + if (digit === undefined || digit > 9) { + return null + } + out += String(digit) + } + return Number(out) +} + +/** + * Consume the longest number at `pos`, either Arabic digits or a run of the + * pack's numeral characters. Digits are capped at four so a year and the month + * that follows it never merge. + */ +export function readNumberAt( + p: P, + source: string, + pos: number, +): { next: number; value: number } | null { + const digits = /^\d{1,4}/.exec(source.slice(pos)) + if (digits) { + return { value: Number(digits[0]), next: pos + digits[0].length } + } + const numerals = p.profile.numerals + if (!numerals) { + return null + } + let end = pos + while (end < source.length && numerals[source[end]!] !== undefined) { + end++ + } + if (end === pos) { + return null + } + const value = readLocaleNumber(p, source.slice(pos, end)) + return value === null ? null : { value, next: end } +} diff --git a/packages/lingo/src/date/parse.ts b/packages/lingo/src/date/parse.ts index 068cd94..7de8f62 100644 --- a/packages/lingo/src/date/parse.ts +++ b/packages/lingo/src/date/parse.ts @@ -26,6 +26,7 @@ import { type SerializedDateRange, } from './serialize' import { type CoreDate, confidence, issue, knownFor, type P, trimRange } from './state' +import { suffixTimeStarts, trimTrailingFillers } from './suffix' import { parseTimeOnly, type TimeCore } from './time' import { TIME_PATTERN } from './vocab' import { applyZoneToCivil, type DateZone, stripTrailingZone } from './zone' @@ -224,6 +225,11 @@ function parseWithOptionalTime(p: P, start: number, end: number): CoreDate | nul } } + const unspaced = parseUnspacedDateTime(p, start, end) + if (unspaced) { + return unspaced + } + const dateOnly = parseDateOnly(p, start, end, true) if (dateOnly) { return dateOnly @@ -260,6 +266,31 @@ function dateTimePattern(p: P): string { return p.profile.date?.timePattern ?? TIME_PATTERN } +/** + * Date and clock written as one unbroken run (`明天下午3点`, `明日の午後3時半`). + * Candidate split points come from the pack's clock vocabulary, so this scan + * only runs for packs that declare one and only tries the few offsets where a + * clock can actually begin. + */ +function parseUnspacedDateTime(p: P, start: number, end: number): CoreDate | null { + const source = p.text.slice(start, end) + for (const at of suffixTimeStarts(p, source)) { + const timeCore = parseTimeOnly(p, start + at, end) + if (!timeCore) { + continue + } + const head = trimTrailingFillers(p, source.slice(0, at)) + if (head.length === 0) { + continue + } + const dateCore = parseDateOnly(p, start, start + head.length, true) + if (dateCore) { + return attachTime(dateCore, timeCore, start, end) + } + } + return null +} + function attachTime(core: CoreDate, time: TimeCore, start: number, end: number): CoreDate { const date = new Date(core.date.getTime()) date.setHours(time.hour, time.minute, time.second, 0) diff --git a/packages/lingo/src/date/relative.ts b/packages/lingo/src/date/relative.ts index 0f8402a..3952aae 100644 --- a/packages/lingo/src/date/relative.ts +++ b/packages/lingo/src/date/relative.ts @@ -14,6 +14,7 @@ import { startOfWeek, withTime, } from './civil' +import { readLocaleNumber } from './numeral' import type { DateAlternative, DateGrain } from './parse' import { type CoreDate, @@ -25,6 +26,7 @@ import { stripDateFillers, trimRange, } from './state' +import { matchAffix, trimLeadingFillers, trimTrailingFillers } from './suffix' import { type DayTimePhrase, DAY_OFFSETS as EN_DAY_OFFSETS, @@ -379,11 +381,13 @@ function parseWeekday(p: P, start: number, end: number): CoreDate | null { function extractWeekdayOffset(p: P, source: string): { days: number; source: string } | null { const phrases = dateWeekdayOffsetPhrases(p) for (const phrase of Object.keys(phrases)) { - if (source.endsWith(` ${phrase}`)) { - return { source: source.slice(0, -phrase.length - 1).trim(), days: phrases[phrase]! } + const trailing = matchAffix(source, phrase, 'end') + if (trailing !== null) { + return { source: trailing.trim(), days: phrases[phrase]! } } - if (source.startsWith(`${phrase} `)) { - return { source: source.slice(phrase.length + 1).trim(), days: phrases[phrase]! } + const leading = matchAffix(source, phrase, 'start') + if (leading !== null) { + return { source: leading.trim(), days: phrases[phrase]! } } } return null @@ -408,11 +412,13 @@ function matchWordModifier( const modifiers = dateModifiers(p) for (const modifier of RELATIVE_MODIFIERS) { for (const modWord of modifiers[modifier]) { - if (source.startsWith(`${modWord} `)) { - return { modifier, word: source.slice(modWord.length + 1) } + const leading = matchAffix(source, modWord, 'start') + if (leading !== null) { + return { modifier, word: trimLeadingFillers(p, leading) } } - if (source.endsWith(` ${modWord}`)) { - return { modifier, word: source.slice(0, -modWord.length - 1) } + const trailing = matchAffix(source, modWord, 'end') + if (trailing !== null) { + return { modifier, word: trailing } } } } @@ -736,11 +742,12 @@ function parseDayPartCompound( ): CoreDate | null { const parts = dateDayPartWords(p) for (const phrase of Object.keys(parts)) { - if (!source.endsWith(` ${phrase}`)) { + const head = matchAffix(source, phrase, 'end') + if (head === null) { continue } - const partStart = end - phrase.length - const { end: anchorEnd } = trimRange(p.text, start, partStart) + const trimmedHead = trimTrailingFillers(p, head) + const { end: anchorEnd } = trimRange(p.text, start, start + trimmedHead.length) if (anchorEnd <= start) { continue } @@ -929,29 +936,10 @@ function parseCompactOffsetBody( if (!unitEntry) { return null } - const value = parseCompactNumber(p, body.slice(0, -unitEntry.length)) + const value = readLocaleNumber(p, body.slice(0, -unitEntry.length)) return value === null ? null : { unit: units[unitEntry]!, value } } -function parseCompactNumber(p: P, text: string): number | null { - if (/^\d+$/.test(text)) { - return Number(text) - } - if (text === '十') { - return 10 - } - const ten = text.indexOf('十') - if (ten >= 0) { - const left = ten === 0 ? 1 : (p.profile.numerals?.[text.slice(0, ten)] ?? Number.NaN) - const right = - ten === text.length - 1 ? 0 : (p.profile.numerals?.[text.slice(ten + 1)] ?? Number.NaN) - const value = left * 10 + right - return Number.isFinite(value) ? value : null - } - const value = p.profile.numerals?.[text] - return value === undefined ? null : value -} - function isJoinWord(p: P, pos: number): boolean { const w = wordAt(p, pos) return w !== null && p.profile.numberWords.andWords.has(w) diff --git a/packages/lingo/src/date/suffix.ts b/packages/lingo/src/date/suffix.ts new file mode 100644 index 0000000..50e89c4 --- /dev/null +++ b/packages/lingo/src/date/suffix.ts @@ -0,0 +1,268 @@ +import type { LingoIssue } from '../core/types' +import { validDateTime } from './civil' +import { readNumberAt } from './numeral' +import type { DateGrain } from './parse' +import { type CoreDate, core, knownFor, needsNow, type P } from './state' +import type { TimeCore } from './time' + +/** + * Grammar for locales that CLOSE each date or clock part with a suffix instead + * of separating parts with a delimiter: `2026年3月5日`, `3月5号`, `15時30分`, + * `下午3点`. Every rule here reads pack data (`date.numericDateSuffixes`, + * `date.clockSuffix`, `date.dayPeriods`, `date.clockMinuteWords`), so packs + * that declare none of it never reach this path. + */ + +interface Read { + next: number + value: number +} + +/** `2026年3月5日`, `3月5号`, `2026年3月`, `2026年`. */ +export function parseSuffixDate(p: P, start: number, end: number): CoreDate | null { + const vocab = p.profile.date?.numericDateSuffixes + if (!vocab) { + return null + } + const source = p.text.slice(start, end) + let pos = 0 + const yearRead = readSuffixed(p, source, pos, vocab.year) + const year = yearRead ? civilYear(yearRead.value) : undefined + if (yearRead) { + if (year === undefined) { + return null + } + pos = yearRead.next + if (pos === source.length) { + return core(new Date(year, 0, 1), 'year', knownFor('year'), start, end) + } + } + const monthRead = readSuffixed(p, source, pos, vocab.month) + if (!monthRead || monthRead.value < 1 || monthRead.value > 12) { + return null + } + pos = monthRead.next + const month = monthRead.value - 1 + if (pos === source.length) { + return dateCore(p, year, month, 1, 'month', start, end) + } + const dayRead = readSuffixed(p, source, pos, vocab.day) + if (!dayRead || dayRead.next !== source.length) { + return null + } + return dateCore(p, year, month, dayRead.value, 'day', start, end) +} + +/** `3点`, `15時30分`, `下午3点半`, `午前9時`, `三点一刻`. */ +export function parseSuffixClock(p: P, source: string, issues: LingoIssue[]): TimeCore | null { + const vocab = p.profile.date?.clockSuffix + if (!vocab) { + return null + } + let pos = 0 + const period = readDayPeriod(p, source, pos) + if (period) { + pos = period.next + } + const hourRead = readSuffixed(p, source, pos, vocab.hour) + if (!hourRead) { + return null + } + pos = hourRead.next + let hour = hourRead.value + let minute = 0 + let second = 0 + let grain: DateGrain = 'hour' + const minuteRead = readSuffixed(p, source, pos, vocab.minute) ?? readMinuteWord(p, source, pos) + if (minuteRead) { + minute = minuteRead.value + grain = 'minute' + pos = minuteRead.next + const secondRead = readSuffixed(p, source, pos, vocab.second) + if (secondRead) { + second = secondRead.value + grain = 'second' + pos = secondRead.next + } + } + if (pos !== source.length) { + return null + } + if (period?.meridiem === 'pm' && hour < 12) { + hour += 12 + } + if (period?.meridiem === 'am' && hour === 12) { + hour = 0 + } + if (hour > 23 || minute > 59 || second > 59) { + return null + } + return { hour, minute, second, grain, issues } +} + +/** + * Split an unspaced date+time compound (`明天下午3点`, `明日の午後3時半`) at the + * head of its clock. Candidate heads are the pack's day-period words and its + * hour markers walked back over the number they close, so the scan is bounded + * by the vocabulary rather than by every offset in the string. + */ +export function suffixTimeStarts(p: P, source: string): number[] { + const date = p.profile.date + const starts = new Set() + for (const word of Object.keys(date?.dayPeriods ?? {})) { + for (let at = source.indexOf(word); at > 0; at = source.indexOf(word, at + 1)) { + starts.add(at) + } + } + for (const marker of date?.clockSuffix?.hour ?? []) { + for (let at = source.indexOf(marker); at > 0; at = source.indexOf(marker, at + 1)) { + const numberStart = walkBackNumber(p, source, at) + if (numberStart > 0) { + starts.add(numberStart) + } + } + } + return [...starts].sort((a, b) => a - b) +} + +/** Drop trailing joiner particles (`明日の` → `明日`) left by an unspaced split. */ +export function trimTrailingFillers(p: P, source: string): string { + return trimFillers(p, source, 'end') +} + +/** Drop leading joiner particles (`の火曜日` → `火曜日`) left by an affix match. */ +export function trimLeadingFillers(p: P, source: string): string { + return trimFillers(p, source, 'start') +} + +function trimFillers(p: P, source: string, side: 'end' | 'start'): string { + const fillers = p.profile.date?.fillerWords + if (!fillers?.length) { + return source + } + let out = source + for (let more = true; more; ) { + more = false + for (const filler of fillers) { + if (filler.length === 0 || out.length <= filler.length || /[a-z]/i.test(filler)) { + continue + } + if (side === 'end' ? out.endsWith(filler) : out.startsWith(filler)) { + out = side === 'end' ? out.slice(0, -filler.length) : out.slice(filler.length) + more = true + } + } + } + return out +} + +/** + * Match an affix against one edge of `source`. Scripts written without word + * spaces glue affixes straight onto the word they modify (`下周五`, + * `明日の午後`), so the separating space is required only for affixes that are + * themselves spelled in a spaced script. + */ +export function matchAffix(source: string, affix: string, side: 'end' | 'start'): string | null { + const spaced = side === 'start' ? `${affix} ` : ` ${affix}` + if (side === 'start' && source.startsWith(spaced)) { + return source.slice(spaced.length) + } + if (side === 'end' && source.endsWith(spaced)) { + return source.slice(0, -spaced.length) + } + if (/[a-z]/i.test(affix)) { + return null + } + if (side === 'start' && source.length > affix.length && source.startsWith(affix)) { + return source.slice(affix.length) + } + if (side === 'end' && source.length > affix.length && source.endsWith(affix)) { + return source.slice(0, -affix.length) + } + return null +} + +function readSuffixed( + p: P, + source: string, + pos: number, + markers: readonly string[] | undefined, +): Read | null { + if (!markers?.length) { + return null + } + const number = readNumberAt(p, source, pos) + if (!number) { + return null + } + const marker = longest(markers, source, number.next) + return marker === null ? null : { value: number.value, next: number.next + marker.length } +} + +/** Minute words that stand alone after the hour: `3時半`, `三点一刻`. */ +function readMinuteWord(p: P, source: string, pos: number): Read | null { + const words = p.profile.date?.clockMinuteWords + if (!words) { + return null + } + const key = longest(Object.keys(words), source, pos) + return key === null ? null : { value: words[key]!, next: pos + key.length } +} + +function readDayPeriod( + p: P, + source: string, + pos: number, +): { meridiem: 'am' | 'pm'; next: number } | null { + const periods = p.profile.date?.dayPeriods + if (!periods) { + return null + } + const key = longest(Object.keys(periods), source, pos) + return key === null ? null : { meridiem: periods[key]!.meridiem, next: pos + key.length } +} + +function longest(candidates: readonly string[], source: string, pos: number): string | null { + let best: string | null = null + for (const candidate of candidates) { + if ( + candidate.length > (best?.length ?? 0) && + source.startsWith(candidate, pos) && + candidate.length > 0 + ) { + best = candidate + } + } + return best +} + +function walkBackNumber(p: P, source: string, at: number): number { + const numerals = p.profile.numerals + let pos = at + while (pos > 0 && (/\d/.test(source[pos - 1]!) || numerals?.[source[pos - 1]!] !== undefined)) { + pos-- + } + return pos === at ? -1 : pos +} + +function dateCore( + p: P, + year: number | undefined, + month: number, + day: number, + grain: DateGrain, + start: number, + end: number, +): CoreDate | null { + const date = validDateTime(year ?? p.now.getFullYear(), month, day, 0, 0, 0) + if (!date) { + return null + } + const parsed = core(date, grain, knownFor(grain), start, end) + return year === undefined ? needsNow(parsed) : parsed +} + +/** Written years are absolute here — `26年` is the 26th year, not 2026. */ +function civilYear(value: number): number | undefined { + return value >= 1 && value <= 9999 ? value : undefined +} diff --git a/packages/lingo/src/date/time.ts b/packages/lingo/src/date/time.ts index 41ee11b..38d8fd7 100644 --- a/packages/lingo/src/date/time.ts +++ b/packages/lingo/src/date/time.ts @@ -1,6 +1,7 @@ import type { LingoIssue } from '../core/types' import type { DateGrain } from './parse' import { issue, type P, trimRange } from './state' +import { parseSuffixClock } from './suffix' import { TIME_ALIASES, type TimeAlias, UNIT_WORDS } from './vocab' import { type DateZone, stripTrailingZone } from './zone' @@ -50,6 +51,11 @@ export function parseTimeCore(p: P, source: string, issues: LingoIssue[]): TimeC return aliasTime(alias, issues) } + const suffixed = parseSuffixClock(p, source, issues) + if (suffixed) { + return suffixed + } + const meridiem = /^(?:(?:at|@)\s*)?(\d{1,2})(?:[:.](\d{2})(?::(\d{2}))?)?\s*(am|pm)$/i.exec( source, ) diff --git a/packages/lingo/src/locale/detect.ts b/packages/lingo/src/locale/detect.ts index 7452195..5bd8b65 100644 --- a/packages/lingo/src/locale/detect.ts +++ b/packages/lingo/src/locale/detect.ts @@ -1,43 +1,73 @@ import { normalizeInput } from '../parse/normalize' -import { tokenize } from '../parse/tokenize' +import { type Token, tokenize } from '../parse/tokenize' import { englishLanguageProfile } from './en-core' import { normalizeLocale, resolveLanguageProfile } from './profile' import type { LanguageProfile, LocalePack } from './types' -const packAliasCache = new WeakMap>() +const packAliasCache = new WeakMap() +const packDetectionCache = new WeakMap>() const englishWordCache = new WeakMap>() +const boundPhraseCache = new WeakMap< + LanguageProfile, + { all: readonly string[]; uniqueVsEnglish: readonly string[] } +>() + +/** + * Normalization + tokenization of the input under test. Detection scores every + * loaded pack against the same input, and the caller (`prepare()`) has usually + * done this work already — sharing one scan keeps detection O(packs) in table + * lookups instead of O(packs) in full re-tokenization. + */ +export interface LocaleScan { + lower: string + raw: string + tokens: readonly Token[] +} + +export function localeScan(input: string): LocaleScan { + const n = normalizeInput(input) + return { lower: n.text.toLowerCase(), raw: input, tokens: tokenize(n) } +} export function detectLanguageProfile( packs: readonly LocalePack[], input: string, + scan?: LocaleScan, ): LanguageProfile { - const locale = detectLocale(packs, input) + const locale = detectLocale(packs, input, scan) return resolveLanguageProfile(packs, locale) } -export function detectLocale(packs: readonly LocalePack[], input: string): string | undefined { - let best: { locale: string; score: number } = { - locale: 'en', - score: scoreProfile(englishLanguageProfile, input), - } +export function detectLocale( + packs: readonly LocalePack[], + input: string, + scan: LocaleScan = localeScan(input), +): string | undefined { + let bestLocale = 'en' + let bestScore = scoreScan(englishLanguageProfile, scan, {}) for (let i = 0; i < packs.length; i++) { - if (normalizeLocale(packs[i]!.locale) === 'en') { + const pack = packs[i]! + if (normalizeLocale(pack.locale) === 'en') { continue } - const profile = resolveLanguageProfile(packs, packs[i]!.locale) - const score = scoreProfile(profile, input, { - aliasWords: unitAliasWords(packs[i]!), - detectionWords: packs[i]!.detectionWords, + const profile = resolveLanguageProfile(packs, pack.locale) + const aliases = unitAliases(pack) + const score = scoreScan(profile, scan, { + aliasPhrases: aliases.phrases, + aliasWords: aliases.words, + detectionWords: packDetectionWords(pack), uniqueAgainst: englishLanguageProfile, }) - if (score > best.score) { - best = { locale: profile.locale, score } + if (score > bestScore) { + bestLocale = profile.locale + bestScore = score } } - return best.score > 0 ? best.locale : undefined + return bestScore > 0 ? bestLocale : undefined } interface ScoreOptions { + aliasPhrases?: readonly string[] aliasWords?: ReadonlySet detectionWords?: readonly string[] | ReadonlySet uniqueAgainst?: LanguageProfile @@ -48,9 +78,11 @@ export function scoreProfile( input: string, options: ScoreOptions = {}, ): number { - const n = normalizeInput(input) - const lower = n.text.toLowerCase() - const tokens = tokenize(n) + return scoreScan(profile, localeScan(input), options) +} + +function scoreScan(profile: LanguageProfile, scan: LocaleScan, options: ScoreOptions): number { + const { lower, tokens } = scan const uniqueAgainst = options.uniqueAgainst ? profileWords(options.uniqueAgainst) : undefined const detectionWordSet = detectionWords(options.detectionWords) let score = 0 @@ -114,19 +146,19 @@ export function scoreProfile( } } - for (const phrase of profile.grammar.boundPhrases) { - if (isUniquePhrase(phrase.phrase, uniqueAgainst) && hasPhrase(lower, phrase.phrase)) { + for (const phrase of boundPhrases(profile, uniqueAgainst)) { + if (hasPhrase(lower, phrase)) { score += 2 } } - for (const alias of options.aliasWords ?? []) { - if (alias.includes(' ') && isUniquePhrase(alias, uniqueAgainst) && hasPhrase(lower, alias)) { + for (const alias of options.aliasPhrases ?? []) { + if (isUniquePhrase(alias, uniqueAgainst) && hasPhrase(lower, alias)) { score += 3 } } if (profile.numerals) { - for (const ch of input) { + for (const ch of scan.raw) { if (profile.numerals[ch] !== undefined) { score += 3 } @@ -136,7 +168,12 @@ export function scoreProfile( return score } -function unitAliasWords(pack: LocalePack): ReadonlySet { +interface PackAliases { + phrases: readonly string[] + words: ReadonlySet +} + +function unitAliases(pack: LocalePack): PackAliases { const cached = packAliasCache.get(pack) if (cached) { return cached @@ -154,13 +191,57 @@ function unitAliasWords(pack: LocalePack): ReadonlySet { words.add(normalizeInput(alias).text.toLowerCase()) } } - packAliasCache.set(pack, words) + const aliases: PackAliases = { + phrases: [...words].filter((alias) => alias.includes(' ')), + words, + } + packAliasCache.set(pack, aliases) + return aliases +} + +/** + * Bound phrases are scanned as substrings on every candidate pack, so the + * uniqueness filter is precomputed per profile rather than re-split per parse. + */ +function boundPhrases( + profile: LanguageProfile, + uniqueAgainst: ReadonlySet | undefined, +): readonly string[] { + let cached = boundPhraseCache.get(profile) + if (!cached) { + const all = profile.grammar.boundPhrases.map((entry) => entry.phrase) + cached = { + all, + uniqueVsEnglish: all.filter((phrase) => + isUniquePhrase(phrase, profileWords(englishLanguageProfile)), + ), + } + boundPhraseCache.set(profile, cached) + } + if (!uniqueAgainst) { + return cached.all + } + if (uniqueAgainst === profileWords(englishLanguageProfile)) { + return cached.uniqueVsEnglish + } + return cached.all.filter((phrase) => isUniquePhrase(phrase, uniqueAgainst)) +} + +const EMPTY_WORDS: ReadonlySet = new Set() + +function packDetectionWords(pack: LocalePack): ReadonlySet { + const cached = packDetectionCache.get(pack) + if (cached) { + return cached + } + const words = detectionWords(pack.detectionWords) + packDetectionCache.set(pack, words) return words } function detectionWords(words: ScoreOptions['detectionWords']): ReadonlySet { if (!words) { - return new Set() + return EMPTY_WORDS } return new Set([...words].map((word) => normalizeInput(word).text.toLowerCase())) } diff --git a/packages/lingo/src/locale/profile.ts b/packages/lingo/src/locale/profile.ts index 234ceaa..4541340 100644 --- a/packages/lingo/src/locale/profile.ts +++ b/packages/lingo/src/locale/profile.ts @@ -1,5 +1,7 @@ import { enCore, englishLanguageProfile } from './en-core' import type { + DateClockSuffixVocab, + DateNumericSuffixVocab, DateVocabPack, GrammarBoundPhrase, GrammarWords, @@ -59,6 +61,14 @@ const DATE_RELATIVE_KEYS = [ 'pastSuffixes', ] as const +/** + * Merging a profile allocates ~40 Sets/Records, so doing it per parse dominates + * the parse itself. Packs are module-level constants held for the lifetime of a + * `createLingo()` instance, so the merge result is cached per pack-array + * identity (weakly) and per requested locale. + */ +const profileCache = new WeakMap>() + export function resolveLanguageProfile( packs: readonly LocalePack[] = DEFAULT_LOCALE_PACKS, locale?: string, @@ -67,9 +77,20 @@ export function resolveLanguageProfile( if (!hasNonEnglishLocalePacks(packs) && matchesLocale(enCore, requested)) { return englishLanguageProfile } + let byLocale = profileCache.get(packs) + if (!byLocale) { + byLocale = new Map() + profileCache.set(packs, byLocale) + } + const cached = byLocale.get(requested) + if (cached) { + return cached + } const all = uniquePacks([...DEFAULT_LOCALE_PACKS, ...packs]) - const pack = findPack(all, locale) ?? findPack(all, 'en')! - return buildProfile(pack, all, new Set()) + const pack = findPack(all, requested) ?? findPack(all, 'en')! + const profile = buildProfile(pack, all, new Set()) + byLocale.set(requested, profile) + return profile } export function hasNonEnglishLocalePacks(packs?: readonly LocalePack[]): boolean { @@ -214,6 +235,7 @@ function mergeDate( calendarPeriodPhrases: mergeRecord(base?.calendarPeriodPhrases, overlay?.calendarPeriodPhrases), clockMinuteWords: mergeRecord(base?.clockMinuteWords, overlay?.clockMinuteWords), clockPastWords: mergeList(base?.clockPastWords ?? [], overlay?.clockPastWords), + clockSuffix: mergeClockSuffix(base?.clockSuffix, overlay?.clockSuffix), clockToWords: mergeList(base?.clockToWords ?? [], overlay?.clockToWords), compactOffset: base?.compactOffset || overlay?.compactOffset @@ -234,11 +256,17 @@ function mergeDate( : undefined, dayOffsets: mergeRecord(base?.dayOffsets, overlay?.dayOffsets), dayPartWords: mergeRecord(base?.dayPartWords, overlay?.dayPartWords), + dayPeriods: mergeRecord(base?.dayPeriods, overlay?.dayPeriods), dayTimePhrases: mergeRecord(base?.dayTimePhrases, overlay?.dayTimePhrases), durationUnitSeconds: mergeRecord(base?.durationUnitSeconds, overlay?.durationUnitSeconds), fillerWords: mergeList(base?.fillerWords ?? [], overlay?.fillerWords), modifiers: mergeListRecord(base?.modifiers, overlay?.modifiers, DATE_MODIFIER_KEYS), months: mergeRecord(base?.months, overlay?.months), + numericDateSuffixes: mergeNumericDateSuffixes( + base?.numericDateSuffixes, + overlay?.numericDateSuffixes, + ), + ordinalSuffixes: mergeList(base?.ordinalSuffixes ?? [], overlay?.ordinalSuffixes), periodEdgePhrases: mergeRecord(base?.periodEdgePhrases, overlay?.periodEdgePhrases), periodWords: mergeListRecord(base?.periodWords, overlay?.periodWords, DATE_PERIOD_KEYS), relative: mergeListRecord(base?.relative, overlay?.relative, DATE_RELATIVE_KEYS), @@ -257,6 +285,34 @@ function mergeRecord(base?: Record, overlay?: Record): return { ...(base ?? {}), ...overlay } } +function mergeClockSuffix( + base?: DateClockSuffixVocab, + overlay?: DateClockSuffixVocab, +): DateClockSuffixVocab | undefined { + if (!(base || overlay)) { + return + } + return { + hour: mergeList(base?.hour ?? [], overlay?.hour), + minute: mergeList(base?.minute ?? [], overlay?.minute), + second: mergeList(base?.second ?? [], overlay?.second), + } +} + +function mergeNumericDateSuffixes( + base?: DateNumericSuffixVocab, + overlay?: DateNumericSuffixVocab, +): DateNumericSuffixVocab | undefined { + if (!(base || overlay)) { + return + } + return { + day: mergeList(base?.day ?? [], overlay?.day), + month: mergeList(base?.month ?? [], overlay?.month), + year: mergeList(base?.year ?? [], overlay?.year), + } +} + function mergeListRecord( base: Partial> | undefined, overlay: Partial> | undefined, diff --git a/packages/lingo/src/locale/types.ts b/packages/lingo/src/locale/types.ts index be1bc05..ae25d5c 100644 --- a/packages/lingo/src/locale/types.ts +++ b/packages/lingo/src/locale/types.ts @@ -14,6 +14,13 @@ export interface GrammarBoundPhrase { bound: 'min' | 'max' exclusive: boolean phrase: string + /** + * The phrase follows the quantity instead of leading it (`5キロ未満`, + * `5公斤以下`). Postposition is a property of the phrase, not of the + * language, so each entry declares it: an unmarked phrase is only ever read + * before the value. + */ + suffix?: boolean } export interface GrammarWords { @@ -161,6 +168,37 @@ export interface DateCompactOffsetVocab { unitWords: Record } +/** + * Clocks whose parts are closed by a suffix rather than separated by a colon: + * `3点`, `3时15分`, `15時30分`. Hour numbers may be digits or, when the pack + * declares `numerals`, its own digits (`三時`). A minute part spelled as a word + * rather than a number (`3時半`, `三点一刻`) comes from `clockMinuteWords`. + */ +export interface DateClockSuffixVocab { + hour: readonly string[] + minute?: readonly string[] + second?: readonly string[] +} + +/** + * A word that fixes a 12-hour clock reading to a half of the day, the way + * am/pm does: `上午`/`下午`, `午前`/`午後`. `am` maps 12 to 0 and leaves 1–11 + * alone; `pm` maps 1–11 to 13–23. + */ +export interface DateDayPeriod { + meridiem: 'am' | 'pm' +} + +/** + * Civil date parts closed by a suffix instead of a separator, as in + * `2026年3月5日` or `3月5号`. + */ +export interface DateNumericSuffixVocab { + day?: readonly string[] + month?: readonly string[] + year?: readonly string[] +} + export interface DateRelativeVocab { anchorWords?: readonly string[] futurePrefixes?: readonly string[] @@ -172,15 +210,26 @@ export interface DateVocabPack { calendarPeriodPhrases?: Record clockMinuteWords?: Record clockPastWords?: readonly string[] + clockSuffix?: DateClockSuffixVocab clockToWords?: readonly string[] compactOffset?: DateCompactOffsetVocab dayOffsets?: Record dayPartWords?: Record + dayPeriods?: Record dayTimePhrases?: Record durationUnitSeconds?: Record fillerWords?: readonly string[] modifiers?: Partial> months: Record + numericDateSuffixes?: DateNumericSuffixVocab + /** + * Ordinal markers that may follow a day number in a written date + * (`1er juillet`, `1º de julho`). Defaults to the English `st/nd/rd/th`. + * Normalization folds the masculine/feminine ordinal indicators `º`/`ª` to + * the degree sign (they are degree look-alikes), so Romance packs list `°` + * next to the ASCII `o`/`a` spellings. + */ + ordinalSuffixes?: readonly string[] periodEdgePhrases?: Record periodWords?: Partial> relative?: DateRelativeVocab diff --git a/packages/lingo/src/locales/es.ts b/packages/lingo/src/locales/es.ts index d0b81c4..e26c91d 100644 --- a/packages/lingo/src/locales/es.ts +++ b/packages/lingo/src/locales/es.ts @@ -145,6 +145,12 @@ export const es: LocalePack = { mil: 1000, millon: 1_000_000, millones: 1_000_000, + millardo: 1_000_000_000, + millardos: 1_000_000_000, + // Spanish is a long-scale language: "billón" is 10^12, NOT the English + // billion. 10^9 is "mil millones" (or the rarer "millardo"). + billon: 1_000_000_000_000, + billones: 1_000_000_000_000, }, tens: { veinte: 20, @@ -210,6 +216,8 @@ export const es: LocalePack = { diciembre: 11, dic: 11, }, + // "1º de julio" / "1er día" — º normalizes to the degree sign. + ordinalSuffixes: ['o', 'a', 'er', '°'], periodEdgePhrases: { principios: { edge: 'start', period: 'month' }, mediados: { edge: 'mid', period: 'month' }, diff --git a/packages/lingo/src/locales/fr.ts b/packages/lingo/src/locales/fr.ts index d5e6bae..3511f60 100644 --- a/packages/lingo/src/locales/fr.ts +++ b/packages/lingo/src/locales/fr.ts @@ -69,6 +69,12 @@ export const fr: LocalePack = { articles: ['un', 'une'], bareScales: { cent: 100, mille: 1000 }, composed: { + // Teens are dix + ones. Without these the tokenizer's "-" between them + // reads "dix-sept" as the range 10-7, so 17/18/19 must match as one unit + // before range splitting sees the hyphen. + 'dix sept': 17, + 'dix huit': 18, + 'dix neuf': 19, 'soixante dix': 70, 'soixante et onze': 71, 'soixante douze': 72, @@ -140,9 +146,6 @@ export const fr: LocalePack = { quatorze: 14, quinze: 15, seize: 16, - dixsept: 17, - dixhuit: 18, - dixneuf: 19, }, scales: { cent: 100, @@ -152,6 +155,10 @@ export const fr: LocalePack = { millions: 1_000_000, milliard: 1_000_000_000, milliards: 1_000_000_000, + // Long scale: French "billion" is 10^12, not the English 10^9 (that is + // "milliard"). + billion: 1_000_000_000_000, + billions: 1_000_000_000_000, }, tens: { vingt: 20, @@ -218,6 +225,8 @@ export const fr: LocalePack = { decembre: 11, dec: 11, }, + // "1er juillet"; the feminine "1re/1ère" is rare for dates but free here. + ordinalSuffixes: ['er', 'ere', 're'], periodEdgePhrases: { debut: { edge: 'start', period: 'month' }, mi: { edge: 'mid', period: 'month' }, diff --git a/packages/lingo/src/locales/ja.ts b/packages/lingo/src/locales/ja.ts index 9738743..cf65943 100644 --- a/packages/lingo/src/locales/ja.ts +++ b/packages/lingo/src/locales/ja.ts @@ -5,10 +5,12 @@ export const ja: LocalePack = { aliases: ['ja-JP'], extends: 'en', defaults: { + // `¥` is shared with the yuan, so the Japanese reading has to be declared. + currency: 'JPY', numberFormat: 'auto', }, units: { - currency: [['JPY', '円']], + currency: [['JPY', '円 えん']], length: [ ['m', 'メートル'], ['cm', 'センチ センチメートル'], @@ -51,11 +53,16 @@ export const ja: LocalePack = { approximateWords: ['約', 'およそ', 'だいたい', 'くらい', 'ぐらい', '暑い'], boundPhrases: [ { phrase: '少なくとも', bound: 'min', exclusive: false }, - { phrase: '以上', bound: 'min', exclusive: false }, - { phrase: 'より多い', bound: 'min', exclusive: true }, { phrase: '最大', bound: 'max', exclusive: false }, - { phrase: '以下', bound: 'max', exclusive: false }, - { phrase: '未満', bound: 'max', exclusive: true }, + { phrase: '最低', bound: 'min', exclusive: false }, + // Japanese bounds are postpositions: they close the quantity. + { phrase: '以上', bound: 'min', exclusive: false, suffix: true }, + { phrase: 'より多い', bound: 'min', exclusive: true, suffix: true }, + { phrase: '超', bound: 'min', exclusive: true, suffix: true }, + { phrase: '以下', bound: 'max', exclusive: false, suffix: true }, + { phrase: '以内', bound: 'max', exclusive: false, suffix: true }, + { phrase: '未満', bound: 'max', exclusive: true, suffix: true }, + { phrase: 'より少ない', bound: 'max', exclusive: true, suffix: true }, ], conversionWords: ['へ', 'に', 'として'], phraseWords: ['から', 'まで', '約', 'およそ', '以上', '以下'], @@ -99,6 +106,9 @@ export const ja: LocalePack = { }, date: { calendarPeriodPhrases: { + 今週: { modifier: 'this', period: 'week' }, + 今月: { modifier: 'this', period: 'month' }, + 今年: { modifier: 'this', period: 'year' }, 来週: { modifier: 'next', period: 'week' }, 来月: { modifier: 'next', period: 'month' }, 来年: { modifier: 'next', period: 'year' }, @@ -110,6 +120,16 @@ export const ja: LocalePack = { 先々月: { modifier: 'beforeLast', period: 'month' }, 去年: { modifier: 'last', period: 'year' }, }, + // `半` is the minute half of an hour here (`3時半`), not the fraction it + // spells in a quantity. + clockMinuteWords: { + 半: 30, + }, + clockSuffix: { + hour: ['時', '时'], + minute: ['分'], + second: ['秒'], + }, compactOffset: { futureSuffixes: ['後'], pastSuffixes: ['前'], @@ -136,6 +156,24 @@ export const ja: LocalePack = { 一昨日: -2, おととい: -2, }, + // Bare day parts, used both alone and after a day word (`明日の朝`). + dayPartWords: { + 朝: { hour: 8 }, + 昼: { hour: 12 }, + 夕方: { hour: 18 }, + 晩: { hour: 19 }, + 夜: { hour: 21 }, + }, + // Halves of the day that fix a 12-hour reading, the way am/pm does. + dayPeriods: { + 午前: { meridiem: 'am' }, + 朝: { meridiem: 'am' }, + 昼: { meridiem: 'pm' }, + 午後: { meridiem: 'pm' }, + 夕方: { meridiem: 'pm' }, + 晩: { meridiem: 'pm' }, + 夜: { meridiem: 'pm' }, + }, dayTimePhrases: { 今朝: { dayOffset: 0, hour: 8 }, 今晩: { dayOffset: 0, hour: 19 }, @@ -143,14 +181,63 @@ export const ja: LocalePack = { 正午: { dayOffset: 0, hour: 12 }, 明日の正午: { dayOffset: 1, hour: 12 }, }, + // The genitive particle joins a date to what follows (`明日の午後`); it is + // droppable the way English "the" is. + fillerWords: ['の'], + modifiers: { + this: ['今週', '今'], + next: ['来週', '来', '次の'], + last: ['先週', '先', '前の'], + afterNext: ['再来週'], + beforeLast: ['先々週'], + }, + months: { + 一月: 0, + 二月: 1, + 三月: 2, + 四月: 3, + 五月: 4, + 六月: 5, + 七月: 6, + 八月: 7, + 九月: 8, + 十月: 9, + 十一月: 10, + 十二月: 11, + }, + numericDateSuffixes: { + day: ['日'], + month: ['月'], + year: ['年'], + }, periodEdgePhrases: { 月末: { edge: 'end', period: 'month' }, + 月初: { edge: 'start', period: 'month' }, + 年末: { edge: 'end', period: 'year' }, + 年始: { edge: 'start', period: 'year' }, 週末: { edge: 'end', period: 'week' }, }, timeAliases: { 正午: { hour: 12 }, 真夜中: { hour: 0 }, }, + weekdayNames: ['日曜日', '月曜日', '火曜日', '水曜日', '木曜日', '金曜日', '土曜日'], + weekdays: { + 月曜日: 1, + 火曜日: 2, + 水曜日: 3, + 木曜日: 4, + 金曜日: 5, + 土曜日: 6, + 日曜日: 0, + 月曜: 1, + 火曜: 2, + 水曜: 3, + 木曜: 4, + 金曜: 5, + 土曜: 6, + 日曜: 0, + }, }, numerals: { 零: 0, diff --git a/packages/lingo/src/locales/pt.ts b/packages/lingo/src/locales/pt.ts index 5349b95..7db4ab9 100644 --- a/packages/lingo/src/locales/pt.ts +++ b/packages/lingo/src/locales/pt.ts @@ -65,7 +65,7 @@ export const pt: LocalePack = { numberWords: { andWords: ['e'], articles: ['um', 'uma'], - bareScales: { cem: 100, mil: 1000 }, + bareScales: { cem: 100, cento: 100, mil: 1000 }, composed: { duzentos: 200, duzentas: 200, @@ -137,8 +137,13 @@ export const pt: LocalePack = { mil: 1000, milhao: 1_000_000, milhoes: 1_000_000, + // Brazilian short scale, the majority reading. European Portuguese + // "bilião" (10^12) is deliberately absent rather than silently wrong — + // see plans/backlog.md. bilhao: 1_000_000_000, bilhoes: 1_000_000_000, + trilhao: 1_000_000_000_000, + trilhoes: 1_000_000_000_000, }, tens: { vinte: 20, @@ -173,7 +178,7 @@ export const pt: LocalePack = { 'meio-dia': { dayOffset: 0, hour: 12 }, 'meio dia': { dayOffset: 0, hour: 12 }, }, - fillerWords: ['o', 'a', 'os', 'as', 'de', 'do', 'da', 'no'], + fillerWords: ['o', 'a', 'os', 'as', 'de', 'do', 'da', 'em', 'no', 'na', 'nos', 'nas'], modifiers: { this: ['este', 'esta'], next: ['proximo', 'proxima', 'que vem'], @@ -204,6 +209,8 @@ export const pt: LocalePack = { dezembro: 11, dez: 11, }, + // "1º de julho" — º normalizes to the degree sign, so both are listed. + ordinalSuffixes: ['o', 'a', '°'], periodEdgePhrases: { inicio: { edge: 'start', period: 'month' }, comeco: { edge: 'start', period: 'month' }, @@ -242,18 +249,30 @@ export const pt: LocalePack = { anos: 'year', }, weekdayNames: ['domingo', 'segunda', 'terca', 'quarta', 'quinta', 'sexta', 'sabado'], + // Monday–Friday are ordinal weekday names whose full form carries "-feira". + // Hyphens survive normalization, so both forms are listed explicitly. weekdays: { domingo: 0, dom: 0, segunda: 1, + 'segunda-feira': 1, + 'segunda feira': 1, seg: 1, terca: 2, + 'terca-feira': 2, + 'terca feira': 2, ter: 2, quarta: 3, + 'quarta-feira': 3, + 'quarta feira': 3, qua: 3, quinta: 4, + 'quinta-feira': 4, + 'quinta feira': 4, qui: 4, sexta: 5, + 'sexta-feira': 5, + 'sexta feira': 5, sex: 5, sabado: 6, sab: 6, diff --git a/packages/lingo/src/locales/zh.ts b/packages/lingo/src/locales/zh.ts index 7542157..b72f8c0 100644 --- a/packages/lingo/src/locales/zh.ts +++ b/packages/lingo/src/locales/zh.ts @@ -5,9 +5,12 @@ export const zh: LocalePack = { aliases: ['zh-CN', 'zh-TW', 'zh-HK', 'zh-SG'], extends: 'en', defaults: { + // `¥` is shared with the yen, so the Chinese reading has to be declared. + currency: 'CNY', numberFormat: 'auto', }, units: { + currency: [['CNY', '元 圆 块 人民币']], length: [ ['m', '米'], ['cm', '厘米'], @@ -52,9 +55,15 @@ export const zh: LocalePack = { { phrase: '至少', bound: 'min', exclusive: false }, { phrase: '不少于', bound: 'min', exclusive: false }, { phrase: '超过', bound: 'min', exclusive: true }, + { phrase: '大于', bound: 'min', exclusive: true }, { phrase: '最多', bound: 'max', exclusive: false }, { phrase: '不超过', bound: 'max', exclusive: false }, { phrase: '少于', bound: 'max', exclusive: true }, + { phrase: '小于', bound: 'max', exclusive: true }, + // Written after the quantity, unlike the phrases above. + { phrase: '以上', bound: 'min', exclusive: false, suffix: true }, + { phrase: '以下', bound: 'max', exclusive: false, suffix: true }, + { phrase: '以内', bound: 'max', exclusive: false, suffix: true }, ], conversionWords: ['到', '为', '成'], phraseWords: ['从', '到', '至', '约', '大约', '至少', '最多'], @@ -101,6 +110,11 @@ export const zh: LocalePack = { }, date: { calendarPeriodPhrases: { + 这周: { modifier: 'this', period: 'week' }, + 这个月: { modifier: 'this', period: 'month' }, + 本周: { modifier: 'this', period: 'week' }, + 本月: { modifier: 'this', period: 'month' }, + 今年: { modifier: 'this', period: 'year' }, 下周: { modifier: 'next', period: 'week' }, 下个月: { modifier: 'next', period: 'month' }, 明年: { modifier: 'next', period: 'year' }, @@ -108,6 +122,18 @@ export const zh: LocalePack = { 上个月: { modifier: 'last', period: 'month' }, 去年: { modifier: 'last', period: 'year' }, }, + // `半` is the minute half of an hour here (`三点半`), not the fraction it + // spells in a quantity; `刻` is the traditional quarter-hour. + clockMinuteWords: { + 半: 30, + 一刻: 15, + 三刻: 45, + }, + clockSuffix: { + hour: ['点', '點', '时', '時'], + minute: ['分', '分钟'], + second: ['秒'], + }, compactOffset: { futureSuffixes: ['后', '以后'], pastSuffixes: ['前'], @@ -128,17 +154,67 @@ export const zh: LocalePack = { }, dayOffsets: { 今天: 0, + 今日: 0, 明天: 1, + 明日: 1, 后天: 2, 大后天: 3, 昨天: -1, + 昨日: -1, 前天: -2, 大前天: -3, }, + // Bare day parts, used both alone and after a day word (`明天早上`). + dayPartWords: { + 凌晨: { hour: 3 }, + 早上: { hour: 8 }, + 早晨: { hour: 8 }, + 上午: { hour: 10 }, + 下午: { hour: 15 }, + 傍晚: { hour: 18 }, + 晚上: { hour: 20 }, + 夜里: { hour: 22 }, + }, + // Halves of the day that fix a 12-hour reading, the way am/pm does. + dayPeriods: { + 凌晨: { meridiem: 'am' }, + 早上: { meridiem: 'am' }, + 早晨: { meridiem: 'am' }, + 上午: { meridiem: 'am' }, + 中午: { meridiem: 'pm' }, + 下午: { meridiem: 'pm' }, + 傍晚: { meridiem: 'pm' }, + 晚上: { meridiem: 'pm' }, + 夜里: { meridiem: 'pm' }, + }, dayTimePhrases: { 中午: { dayOffset: 0, hour: 12 }, 明天中午: { dayOffset: 1, hour: 12 }, }, + modifiers: { + this: ['这', '这个', '本'], + next: ['下', '下个'], + last: ['上', '上个'], + }, + months: { + 一月: 0, + 二月: 1, + 三月: 2, + 四月: 3, + 五月: 4, + 六月: 5, + 七月: 6, + 八月: 7, + 九月: 8, + 十月: 9, + 十一月: 10, + 十二月: 11, + }, + numericDateSuffixes: { + day: ['日', '号', '號'], + month: ['月'], + year: ['年'], + }, periodEdgePhrases: { 月初: { edge: 'start', period: 'month' }, 月中: { edge: 'mid', period: 'month' }, @@ -151,6 +227,33 @@ export const zh: LocalePack = { 中午: { hour: 12 }, 午夜: { hour: 0 }, }, + weekdayNames: ['星期日', '星期一', '星期二', '星期三', '星期四', '星期五', '星期六'], + weekdays: { + 星期一: 1, + 星期二: 2, + 星期三: 3, + 星期四: 4, + 星期五: 5, + 星期六: 6, + 星期日: 0, + 星期天: 0, + 周一: 1, + 周二: 2, + 周三: 3, + 周四: 4, + 周五: 5, + 周六: 6, + 周日: 0, + 周天: 0, + 礼拜一: 1, + 礼拜二: 2, + 礼拜三: 3, + 礼拜四: 4, + 礼拜五: 5, + 礼拜六: 6, + 礼拜日: 0, + 礼拜天: 0, + }, }, numerals: { 零: 0, diff --git a/packages/lingo/src/number/value.ts b/packages/lingo/src/number/value.ts index 8cfab93..2837338 100644 --- a/packages/lingo/src/number/value.ts +++ b/packages/lingo/src/number/value.ts @@ -15,6 +15,12 @@ import { parseAndFractionTail, parseNumberWords } from './words' export interface ValueCtx { kind?: Kind n: Normalized + /** + * Inside "between A and B" the and-word belongs to the range, so the A-side + * must not read it as number composition ("between one thousand and two + * thousand"). + */ + noAnd?: boolean numberFormat: NumberFormatPolicy numberWords?: boolean profile?: LanguageProfile @@ -118,7 +124,7 @@ export function parseValue(ctx: ValueCtx, i: number, atStart = false): ValueNode } } } - const wn = parseNumberWords(tokens, i, ctx.profile?.numberWords) + const wn = parseNumberWords(tokens, i, ctx.profile?.numberWords, ctx.noAnd) if (!wn) { return null } diff --git a/packages/lingo/src/number/words-romance.test.ts b/packages/lingo/src/number/words-romance.test.ts index c58fa95..82891cd 100644 --- a/packages/lingo/src/number/words-romance.test.ts +++ b/packages/lingo/src/number/words-romance.test.ts @@ -620,3 +620,184 @@ describe('F5: approximatePhrases longest-first + trailing variants', () => { } }) }) + +// ─── F6: hundreds bind to the group in front of them ──────────────────────── + +describe('F6: hundreds multiply only the preceding 1..99 group', () => { + const value = (input: string, pack: LocalePack) => { + const r = qty(input, pack) + expect(r.ok, JSON.stringify(r.ok ? [] : r.issues)).toBe(true) + return r.ok ? r.quantity.value : Number.NaN + } + + it('"mille cinq cents kg" = 1500, not 1005 x 100 (FR)', () => { + expect(value('mille cinq cents kg', fr)).toBe(1500) + }) + + it('"deux cent cinquante mille kg" = 250,000 (FR)', () => { + expect(value('deux cent cinquante mille kg', fr)).toBe(250_000) + }) + + it('"cent vingt kg" = 120 (FR)', () => { + expect(value('cent vingt kg', fr)).toBe(120) + }) + + it('"nineteen hundred kg" = 1900 (EN, unchanged)', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('nineteen hundred kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(1900) + } + }) + + it('"one thousand five hundred kg" = 1500 (EN, unchanged)', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('one thousand five hundred kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(1500) + } + }) +}) + +// ─── F7: scale chaining ("mil millones") ──────────────────────────────────── + +describe('F7: a banked smaller scale multiplies the next one', () => { + const value = (input: string, pack: LocalePack) => { + const r = qty(input, pack) + expect(r.ok, JSON.stringify(r.ok ? [] : r.issues)).toBe(true) + return r.ok ? r.quantity.value : Number.NaN + } + + it('"mil millones de kg" = 10^9, not 1,001,000 (ES)', () => { + expect(value('mil millones de kg', es)).toBe(1e9) + }) + + it('"dos mil millones de kg" = 2 x 10^9 (ES)', () => { + expect(value('dos mil millones de kg', es)).toBe(2e9) + }) + + it('"mil milhoes de kg" = 10^9 (PT)', () => { + expect(value('mil milhoes de kg', pt)).toBe(1e9) + }) + + it('"mille millions de kg" = 10^9 (FR)', () => { + expect(value('mille millions de kg', fr)).toBe(1e9) + }) + + it('a larger banked scale still adds: "dos millones mil kg" = 2,001,000 (ES)', () => { + expect(value('dos millones mil kg', es)).toBe(2_001_000) + }) + + it('"two hundred thousand kg" = 200,000 (EN, unchanged)', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('two hundred thousand kg') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(200_000) + } + }) +}) + +// ─── F8: and-word links a bare scale to its remainder ─────────────────────── + +describe('F8: bare scale + and-word + remainder', () => { + it('"cento e vinte kg" = 120 (PT)', () => { + const r = qty('cento e vinte kg', pt) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(120) + } + }) + + it('"mil e quinhentos kg" = 1500 (PT)', () => { + const r = qty('mil e quinhentos kg', pt) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(1500) + } + }) + + // GUARD: a scale word after the and-word is a range side, not a remainder. + it('"entre cien y mil metros" stays a range (ES)', () => { + const r = range('entre cien y mil metros', es) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.range.min()?.value).toBe(100) + expect(r.range.max()?.value).toBe(1000) + } + }) + + // GUARD: the fraction tail still owns the and-word. + it('"mil e meio metros" = 1000.5 (PT)', () => { + const r = qty('mil e meio metros', pt) + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(1000.5) + } + }) +}) + +// ─── F9: "between A and B" with spelled scale words on both sides ──────────── + +describe('F9: the range and-word wins over number composition', () => { + const bounds = (input: string, pack: LocalePack) => { + const r = range(input, pack) + expect(r.ok, JSON.stringify(r.ok ? [] : r.issues)).toBe(true) + return r.ok ? [r.range.min()?.value, r.range.max()?.value] : [] + } + + it('"between one thousand and two thousand meters" = 1000..2000 (EN)', () => { + const lingo = createLingo() + const r = lingo.parseRange('between one thousand and two thousand meters') + expect(r.ok, JSON.stringify(r.ok ? [] : r.issues)).toBe(true) + if (r.ok) { + expect(r.range.min()?.value).toBe(1000) + expect(r.range.max()?.value).toBe(2000) + } + }) + + it('"between two hundred and five hundred meters" = 200..500 (EN)', () => { + const lingo = createLingo() + const r = lingo.parseRange('between two hundred and five hundred meters') + expect(r.ok, JSON.stringify(r.ok ? [] : r.issues)).toBe(true) + if (r.ok) { + expect(r.range.min()?.value).toBe(200) + expect(r.range.max()?.value).toBe(500) + } + }) + + it('"entre mille et deux mille metres" = 1000..2000 (FR)', () => { + expect(bounds('entre mille et deux mille metres', fr)).toEqual([1000, 2000]) + }) + + it('"entre mil y dos mil metros" = 1000..2000 (ES)', () => { + expect(bounds('entre mil y dos mil metros', es)).toEqual([1000, 2000]) + }) + + it('"entre mil e dois mil metros" = 1000..2000 (PT)', () => { + expect(bounds('entre mil e dois mil metros', pt)).toEqual([1000, 2000]) + }) + + // GUARD: the fraction tail still binds tighter than the range separator. + it('"between five and a half and ten kg" = 5.5..10 (EN)', () => { + const lingo = createLingo() + const r = lingo.parseRange('between five and a half and ten kg') + expect(r.ok, JSON.stringify(r.ok ? [] : r.issues)).toBe(true) + if (r.ok) { + expect(r.range.min()?.value).toBe(5.5) + expect(r.range.max()?.value).toBe(10) + } + }) + + // GUARD: outside a "between", the and-word still composes numbers. + it('"two hundred and five meters" is still 205 (EN)', () => { + const lingo = createLingo() + const r = lingo.parseQuantity('two hundred and five meters') + expect(r.ok).toBe(true) + if (r.ok) { + expect(r.quantity.value).toBe(205) + } + }) +}) diff --git a/packages/lingo/src/number/words.ts b/packages/lingo/src/number/words.ts index 2163056..86868f3 100644 --- a/packages/lingo/src/number/words.ts +++ b/packages/lingo/src/number/words.ts @@ -34,6 +34,7 @@ export function parseNumberWords( tokens: Token[], i: number, tables: NumberWordTables = EN_NUMBER_WORDS, + noAnd = false, ): WordNumberResult | null { let pos = i let negative = false @@ -43,7 +44,7 @@ export function parseNumberWords( pos++ } - const result = parseCore(tokens, pos, tables) + const result = parseCore(tokens, pos, tables, noAnd) if (!result) { return null } @@ -56,7 +57,12 @@ export function parseNumberWords( return result } -function parseCore(tokens: Token[], i: number, tables: NumberWordTables): WordNumberResult | null { +function parseCore( + tokens: Token[], + i: number, + tables: NumberWordTables, + noAnd: boolean, +): WordNumberResult | null { const pos = i const w = word(tokens[pos]) if (w === null) { @@ -67,7 +73,7 @@ function parseCore(tokens: Token[], i: number, tables: NumberWordTables): WordNu if (tables.articles.has(w)) { const nextW = word(tokens[pos + 1]) if (nextW && tables.scales[nextW] !== undefined) { - return finishScaled(tokens, pos + 1, 1, tables) + return finishScaled(tokens, pos + 1, 1, tables, noAnd) } if (nextW && tables.dozenWords.has(nextW)) { return withHalfDozenTail(tokens, pos + 2, 12, tables) @@ -125,16 +131,22 @@ function parseCore(tokens: Token[], i: number, tables: NumberWordTables): WordNu // Composed table: longest-match phrase lookup (quinientos, quatre-vingt-dix, etc.) const comp = tryComposed(tokens, pos, tables) if (comp) { - return cardinalLoop(tokens, comp.next, 0, comp.value, true, tables) + return cardinalLoop(tokens, comp.next, 0, comp.value, true, tables, noAnd) } - // Bare scale words: "cien gramos", "mil metros" - if (tables.bareScales?.[w] !== undefined) { - return cardinalLoop(tokens, pos + 1, 0, tables.bareScales[w]!, true, tables) + // Bare scale words: "cien gramos", "mil metros". A thousand and up seeds the + // total so a following hundreds group stays its own term ("mille cinq + // cents" = 1000 + 500, not 1005 × 100). + const bare = tables.bareScales?.[w] + if (bare !== undefined) { + const next = eatAndLink(tokens, pos + 1, tables, noAnd) + return bare >= 1000 + ? cardinalLoop(tokens, next, bare, 0, true, tables, noAnd) + : cardinalLoop(tokens, next, 0, bare, true, tables, noAnd) } // Standard cardinal grammar. - return cardinalLoop(tokens, pos, 0, 0, false, tables) + return cardinalLoop(tokens, pos, 0, 0, false, tables, noAnd) } /** Longest-match against composed table; hyphens transparent. */ @@ -178,6 +190,7 @@ function cardinalLoop( current: number, any: boolean, tables: NumberWordTables, + noAnd: boolean, ): WordNumberResult | null { let pos = startPos let t_ = total @@ -212,7 +225,7 @@ function cardinalLoop( const skip = nxt && nxt.type === 'sym' && nxt.text === '-' ? 1 - : word(tokens[pos]) && tables.andWords.has(word(tokens[pos])!) + : !noAnd && word(tokens[pos]) && tables.andWords.has(word(tokens[pos])!) ? 1 : 0 if (skip) { @@ -231,25 +244,21 @@ function cardinalLoop( continue } if (tables.scales[tw] !== undefined && any) { - if (tables.scales[tw] === 100) { - c_ = (c_ === 0 ? 1 : c_) * 100 + const scale = tables.scales[tw]! + if (scale === 100) { + // Hundreds multiply only the 1..99 group in front of them; anything + // already accumulated above that is a separate term. + const group = c_ % 100 + c_ += group === 0 ? 100 : group * 99 + } else if (c_ === 0 && t_ > 0 && t_ < scale) { + // Scale chaining: "mil millones" (10^9), "dos mil millones" — the + // smaller scale already banked in the total is the multiplicand. + t_ *= scale } else { - t_ += (c_ === 0 ? 1 : c_) * tables.scales[tw]! + t_ += (c_ === 0 ? 1 : c_) * scale c_ = 0 } - pos++ - const j = word(tokens[pos]) - if (j && tables.andWords.has(j)) { - const a = word(tokens[pos + 1]) - if ( - a && - (tables.ones[a] !== undefined || - tables.tens[a] !== undefined || - tables.composed?.[a] !== undefined) - ) { - pos++ - } - } + pos = eatAndLink(tokens, eatScaleLink(tokens, pos + 1, scale, tables), tables, noAnd) continue } break @@ -301,6 +310,8 @@ function cardinalLoop( if (dozen) { return withHalfDozenTail(tokens, pos, value, tables) } + // The fraction tail keeps the and-word even under `noAnd`: in "between five + // and a half and ten kg" the first join builds 5.5, the second is the range. const tail = parseAndFractionTail(tokens, pos, tables) if (tail) { value += tail.add @@ -314,14 +325,60 @@ function finishScaled( scalePos: number, multiplier: number, tables: NumberWordTables, + noAnd: boolean, ): WordNumberResult { const scaleWord = word(tokens[scalePos])! - const scaleValue = multiplier * tables.scales[scaleWord]! - const pos = scalePos + 1 + const scale = tables.scales[scaleWord]! + const pos = eatScaleLink(tokens, scalePos + 1, scale, tables) // Continue into cardinalLoop to consume further composed/scale chains. // E.g. "un millon quinientos mil" seeds total with 1,000,000, then // cardinalLoop picks up "quinientos mil" (500*1000). - return cardinalLoop(tokens, pos, scaleValue, 0, true, tables)! + return cardinalLoop(tokens, pos, multiplier * scale, 0, true, tables, noAnd)! +} + +/** + * Romance languages link big scales to the noun they count with "of": "un + * millón de euros", "un million de kilos", "um milhão de quilos". Smaller + * scales do not ("mil euros"), and below a million "de" is far more likely to + * be opening a range ("de 5 a 10"), so the link is only eaten from 10^6 up and + * only when something follows it. + */ +function eatScaleLink( + tokens: Token[], + pos: number, + scale: number, + tables: NumberWordTables, +): number { + if (scale < 1_000_000) { + return pos + } + const link = word(tokens[pos]) + return link && tables.ofWords.has(link) && word(tokens[pos + 1]) !== null ? pos + 1 : pos +} + +/** + * "two hundred and five", "mil e quinhentos", "cento e vinte" — the and-word + * joining a scale to its remainder is punctuation. It is only skipped when a + * plain number word follows: "and a half" belongs to the fraction tail, and a + * following scale word ("entre cien y mil") is a range side, not a remainder. + */ +function eatAndLink( + tokens: Token[], + pos: number, + tables: NumberWordTables, + noAnd: boolean, +): number { + const j = word(tokens[pos]) + if (noAnd || !(j && tables.andWords.has(j))) { + return pos + } + const a = word(tokens[pos + 1]) + const known = + a !== null && + (tables.ones[a] !== undefined || + tables.tens[a] !== undefined || + tables.composed?.[a] !== undefined) + return known ? pos + 1 : pos } /** diff --git a/packages/lingo/src/parse/config.ts b/packages/lingo/src/parse/config.ts index aa01395..9bcc694 100644 --- a/packages/lingo/src/parse/config.ts +++ b/packages/lingo/src/parse/config.ts @@ -23,7 +23,7 @@ import type { LanguageProfile, LocalePack } from '../locale/types' import { parseValue, type ValueCtx } from '../number/value' import { type Normalized, normalizeInput, toSourceSpan } from './normalize' import type { SerializedResult } from './serialize' -import { type Token, tokenize } from './tokenize' +import { splitGluedWords, type Token, tokenize } from './tokenize' /** * Options shared by `lingo()`/`parseQuantity()`/`parseRange()`/`partialState()` @@ -318,6 +318,40 @@ export function resolveConfig(opts: ParseOptions): ParserConfig { } } +const gluedVocabularyCache = new WeakMap() + +/** + * Grammar words a space-free script can glue onto its neighbours. Only + * non-Latin entries qualify: in a spaced language the space already marks the + * boundary, and cutting inside a Latin word would invent tokens that were never + * written. Sorted longest-first so `不超过` wins over the `超过` inside it. + */ +function gluedWordVocabulary(profile: LanguageProfile): readonly string[] { + const cached = gluedVocabularyCache.get(profile) + if (cached) { + return cached + } + const g = profile.grammar + const vocabulary = [ + ...g.boundPhrases.map((entry) => entry.phrase), + ...g.rangeSeparatorWords, + ...g.rangeAndWords, + ...g.rangeAlternativeWords, + ...g.rangeBetweenWords, + ...g.rangeFromWords, + ...g.conversionWords, + ...g.approximateWords, + ...g.approximatePhrases, + ...g.trailingApproxWords, + ...g.trailingApproxPhrases, + ] + .filter((word) => word.length > 0 && !/[a-z]/i.test(word)) + .sort((a, b) => b.length - a.length) + const unique = [...new Set(vocabulary)] + gluedVocabularyCache.set(profile, unique) + return unique +} + export function prepare(input: string, opts: ParseOptions): ParserState { if (!opts.registry) { throw new Error( @@ -326,6 +360,8 @@ export function prepare(input: string, opts: ParseOptions): ParserState { } const n = normalizeInput(input) const text = n.text + const lower = text.toLowerCase() + const tokens = tokenize(n) const localePacks = opts.localePacks const localeNotLoaded = opts.locale && !isLocaleLoaded(localePacks, opts.locale) @@ -334,14 +370,19 @@ export function prepare(input: string, opts: ParseOptions): ParserState { const profile = opts.locale ? resolveLanguageProfile(localePacks, opts.locale) : hasNonEnglishLocalePacks(localePacks) - ? detectLanguageProfile(localePacks!, input) + ? // Detection reuses this pass instead of re-normalizing per loaded pack. + detectLanguageProfile(localePacks!, input, { lower, raw: input, tokens }) : englishLanguageProfile return { src: input, n, text, - lower: text.toLowerCase(), - tokens: tokenize(n), + lower, + tokens: splitGluedWords( + tokens, + gluedWordVocabulary(profile), + (at) => opts.registry!.matchUnitsAt(text, lower, at, opts.kind)[0]?.length ?? 0, + ), reg: opts.registry, opts, profile, @@ -415,7 +456,11 @@ export function tokenAfter(p: ParserState, pos: number): number { * The one place a `ValueCtx` is assembled from parser state — call sites only * choose the `kind` fallback (`expectKind ?? p.opts.kind`, `a.kind ?? …`). */ -export function valueCtx(p: ParserState, kind: Kind | undefined = p.opts.kind): ValueCtx { +export function valueCtx( + p: ParserState, + kind: Kind | undefined = p.opts.kind, + noAnd = false, +): ValueCtx { return { tokens: p.tokens, n: p.n, @@ -424,6 +469,7 @@ export function valueCtx(p: ParserState, kind: Kind | undefined = p.opts.kind): kind, numberWords: p.config.numberWords, profile: p.profile, + noAnd, } } diff --git a/packages/lingo/src/parse/quantity.ts b/packages/lingo/src/parse/quantity.ts index 7d5f5e4..cdd15ee 100644 --- a/packages/lingo/src/parse/quantity.ts +++ b/packages/lingo/src/parse/quantity.ts @@ -34,6 +34,9 @@ export interface Quals { /** True when a bound phrase ("under", "over", "at least"…) begins at `pos`. */ function startsBound(p: ParserState, pos: number): boolean { for (const phrase of p.profile.grammar.boundPhrases) { + if (phrase.suffix) { + continue + } const nx = eatPhrase(p, pos, phrase.phrase) if (nx >= 0) { // "min"/"max" only count as bounds when a value follows. @@ -46,6 +49,27 @@ function startsBound(p: ParserState, pos: number): boolean { return false } +/** + * A bound phrase that FOLLOWS the quantity (`5キロ未満`, `5公斤以上`). Only + * phrases the pack marks `suffix` are read here, so a leading-only phrase can + * never be mistaken for a postposition. + */ +export function parseTrailingBound( + p: ParserState, + pos: number, +): { bound: BoundQual; next: number } | null { + for (const phrase of p.profile.grammar.boundPhrases) { + if (!phrase.suffix) { + continue + } + const next = eatPhrase(p, pos, phrase.phrase) + if (next >= 0) { + return { bound: { bound: phrase.bound, exclusive: phrase.exclusive }, next } + } + } + return null +} + export function parseQualifiers(p: ParserState, i: number): Quals { let pos = i let approximate = false @@ -136,6 +160,9 @@ export function parseQualifiers(p: ParserState, i: number): Quals { } let matched = false for (const phrase of p.profile.grammar.boundPhrases) { + if (phrase.suffix) { + continue + } const nx = eatPhrase(p, pos, phrase.phrase) if (nx >= 0) { // "min"/"max" only count as qualifiers when a value follows. @@ -286,7 +313,7 @@ export function parseQty( expectKind?: Kind, flags?: QtyFlags, ): QtyNode | null { - const ctx = valueCtx(p, expectKind ?? p.opts.kind) + const ctx = valueCtx(p, expectKind ?? p.opts.kind, flags?.noAnd) const prefix = parseCurrencyPrefixQty(p, i, atStart, expectKind, ctx) if (prefix) { return prefix diff --git a/packages/lingo/src/parse/range.ts b/packages/lingo/src/parse/range.ts index 103575b..7093405 100644 --- a/packages/lingo/src/parse/range.ts +++ b/packages/lingo/src/parse/range.ts @@ -28,6 +28,7 @@ import { minorCurrencyScale, parseQty as parseQtyBase, parseQualifiers, + parseTrailingBound, type QtyFlags, type QtyNode, type QualifiedQty, @@ -80,24 +81,27 @@ export function parseRangeOrQty(p: ParserState, i: number, atStart: boolean): Pa return { result: fail(p), nextToken: a.nextToken } } - // Open bound from qualifier: "under 10 min". - if (quals.bound) { + // Open bound, from a leading qualifier ("under 10 min") or a trailing one + // ("5キロ未満") — the same range either way, only the word order differs. + const trailing = quals.bound ? null : parseTrailingBound(p, a.nextToken) + const openBound = quals.bound ?? trailing?.bound + if (openBound) { const withUnit = ensureUnit(p, a) if (!withUnit) { return numberish(p, a, quals.approximate) } const { kind, base, unitId } = withUnit - const bound = quals.bound + const end = trailing ? p.tokens[trailing.next - 1]!.end : a.normEnd const range = new QuantityRange(p.reg, kind, { - min: bound.bound === 'min' ? { base, unit: unitId } : undefined, - max: bound.bound === 'max' ? { base, unit: unitId } : undefined, - exclusiveMin: bound.bound === 'min' ? bound.exclusive : false, - exclusiveMax: bound.bound === 'max' ? bound.exclusive : false, + min: openBound.bound === 'min' ? { base, unit: unitId } : undefined, + max: openBound.bound === 'max' ? { base, unit: unitId } : undefined, + exclusiveMin: openBound.bound === 'min' ? openBound.exclusive : false, + exclusiveMax: openBound.bound === 'max' ? openBound.exclusive : false, approximate: quals.approximate || a.approximate, }) return { - result: okRange(p, range, i, a.normEnd), - nextToken: a.nextToken, + result: okRange(p, range, i, end), + nextToken: trailing?.next ?? a.nextToken, } } diff --git a/packages/lingo/src/parse/tokenize.ts b/packages/lingo/src/parse/tokenize.ts index 302c858..a18d560 100644 --- a/packages/lingo/src/parse/tokenize.ts +++ b/packages/lingo/src/parse/tokenize.ts @@ -110,6 +110,85 @@ export function tokenize(n: Normalized): Token[] { return tokens } +/** How many characters of a unit alias start at this offset in the token. */ +export type AliasLengthAt = (start: number) => number + +/** + * Scripts written without word spaces hand the tokenizer one long run + * (`三至五公斤`, `5キロ未満`), which hides the grammar words inside it: the + * range separator never matches, and a bound or approximant glued behind the + * unit is silently swallowed with the rest of the token. Cutting the run at the + * profile's own grammar words restores the token boundaries the grammar + * expects, so no per-language parser branch is needed. + * + * Only non-Latin vocabulary splits, so spaced languages tokenize unchanged. + */ +export function splitGluedWords( + tokens: Token[], + vocabulary: readonly string[], + aliasLengthAt: AliasLengthAt, +): Token[] { + if (vocabulary.length === 0) { + return tokens + } + let out: Token[] | null = null + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i]! + const pieces = token.type === 'word' ? cutAtVocabulary(token, vocabulary, aliasLengthAt) : null + if (pieces) { + out ??= tokens.slice(0, i) + out.push(...pieces) + } else { + out?.push(token) + } + } + return out ?? tokens +} + +function cutAtVocabulary( + token: Token, + vocabulary: readonly string[], + aliasLengthAt: AliasLengthAt, +): Token[] | null { + // A token that IS a grammar word stays whole: `不超过` must not be cut into + // `不` + the `超过` nested inside it. + if (token.text.length < 2 || isLetter(token.text[0]!) || vocabulary.includes(token.text)) { + return null + } + const cuts: number[] = [] + for (let at = 0; at < token.text.length; ) { + const hit = vocabulary.find( + (word) => word.length <= token.text.length - at && token.text.startsWith(word, at), + ) + // A unit alias is a word too: never cut inside one, or `一時間半` loses its + // hour to the `間` that Japanese also uses as a range word. + const alias = aliasLengthAt(token.start + at) + if (hit && hit.length >= alias) { + cuts.push(at, at + hit.length) + at += hit.length + } else { + at += Math.max(alias, 1) + } + } + if (cuts.length === 0) { + return null + } + const bounds = [...new Set([0, ...cuts, token.text.length])].sort((a, b) => a - b) + const pieces: Token[] = [] + for (let i = 1; i < bounds.length; i++) { + const from = bounds[i - 1]! + const to = bounds[i]! + pieces.push({ + type: 'word', + text: token.text.slice(from, to), + start: token.start + from, + end: token.start + to, + spaceBefore: i === 1 ? token.spaceBefore : false, + }) + } + return pieces +} + function sameOriginFraction(n: Normalized, i: number): boolean { if (n.start.length === 0) { return false diff --git a/packages/lingo/tests/corpus/contract-v1.json b/packages/lingo/tests/corpus/contract-v1.json index 035e7e0..c516420 100644 --- a/packages/lingo/tests/corpus/contract-v1.json +++ b/packages/lingo/tests/corpus/contract-v1.json @@ -2539,6 +2539,89 @@ }, "confidence": 1 }, + "between one thousand and two thousand meters": { + "input": "between one thousand and two thousand meters", + "type": "range", + "kind": "length", + "base": { + "min": 1000, + "max": 2000 + }, + "unit": { + "min": "m", + "max": "m" + }, + "issues": [], + "span": { + "start": 8, + "end": 44 + }, + "confidence": 1 + }, + "between two hundred and five hundred meters": { + "input": "between two hundred and five hundred meters", + "type": "range", + "kind": "length", + "base": { + "min": 200, + "max": 500 + }, + "unit": { + "min": "m", + "max": "m" + }, + "issues": [], + "span": { + "start": 8, + "end": 43 + }, + "confidence": 1 + }, + "between five and a half and ten kg": { + "input": "between five and a half and ten kg", + "type": "range", + "kind": "mass", + "base": { + "min": 5.5, + "max": 10 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 8, + "end": 34 + }, + "confidence": 1 + }, + "nineteen hundred kg": { + "input": "nineteen hundred kg", + "type": "quantity", + "kind": "mass", + "base": 1900, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 19 + }, + "confidence": 1 + }, + "two hundred thousand kg": { + "input": "two hundred thousand kg", + "type": "quantity", + "kind": "mass", + "base": 200000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 23 + }, + "confidence": 1 + }, "": { "input": "", "type": "fail", diff --git a/packages/lingo/tests/corpus/locale-es-contract-v1.json b/packages/lingo/tests/corpus/locale-es-contract-v1.json index ea5b1e2..eb6ecbb 100644 --- a/packages/lingo/tests/corpus/locale-es-contract-v1.json +++ b/packages/lingo/tests/corpus/locale-es-contract-v1.json @@ -662,6 +662,146 @@ "end": 12 }, "confidence": 0.9 + }, + "ciento veinte kg": { + "input": "ciento veinte kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 120, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 1 + }, + "un millon de kg": { + "input": "un millon de kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 1000000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + }, + "mil millones de kg": { + "input": "mil millones de kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 1000000000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 18 + }, + "confidence": 1 + }, + "dos mil millones de kg": { + "input": "dos mil millones de kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 2000000000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 22 + }, + "confidence": 1 + }, + "un billon de kg": { + "input": "un billon de kg", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "mass", + "base": 1000000000000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + }, + "20 euros": { + "input": "20 euros", + "opts": { + "locale": "es" + }, + "type": "quantity", + "kind": "currency", + "base": 20, + "unit": "EUR", + "issues": [], + "span": { + "start": 0, + "end": 8 + }, + "confidence": 1 + }, + "entre mil y dos mil kg": { + "input": "entre mil y dos mil kg", + "opts": { + "locale": "es" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 1000, + "max": 2000 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 6, + "end": 22 + }, + "confidence": 1 + }, + "entre cien y mil kg": { + "input": "entre cien y mil kg", + "opts": { + "locale": "es" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 100, + "max": 1000 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 6, + "end": 19 + }, + "confidence": 1 } }, "date": { @@ -940,6 +1080,98 @@ "end": 18 }, "confidence": 1 + }, + "el 1º de marzo": { + "input": "el 1º de marzo", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2027, + "month": 3, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 14 + }, + "confidence": 1 + }, + "el 3 de marzo": { + "input": "el 3 de marzo", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2027, + "month": 3, + "day": 3, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "el proximo lunes": { + "input": "el proximo lunes", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 6, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 1 + }, + "lunes que viene": { + "input": "lunes que viene", + "opts": { + "locale": "es" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 6, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 } }, "dateRange": {} diff --git a/packages/lingo/tests/corpus/locale-es-source.mjs b/packages/lingo/tests/corpus/locale-es-source.mjs index 624bf01..762cef7 100644 --- a/packages/lingo/tests/corpus/locale-es-source.mjs +++ b/packages/lingo/tests/corpus/locale-es-source.mjs @@ -52,6 +52,18 @@ export const breadthRows = [ ['de 5 a 10 kg', { locale: 'es' }], // fuzzyAmounts ['un par de kg', { locale: 'es' }], + // Hundreds attach to the group in front of them, not the running total + ['ciento veinte kg', { locale: 'es' }], + // Long scale: "mil millones" is 10^9, "billón" is 10^12 + ['un millon de kg', { locale: 'es' }], + ['mil millones de kg', { locale: 'es' }], + ['dos mil millones de kg', { locale: 'es' }], + ['un billon de kg', { locale: 'es' }], + // Currency + ['20 euros', { locale: 'es' }], + // The range and-word wins over number composition on both sides + ['entre mil y dos mil kg', { locale: 'es' }], + ['entre cien y mil kg', { locale: 'es' }], ] export const dateRows = [ @@ -71,6 +83,11 @@ export const dateRows = [ ['a mediados de julio', { locale: 'es' }], ['a finales de mes', { locale: 'es' }], ['a finales de julio', { locale: 'es' }], + // Day-of-month, ordinal and cardinal + ['el 1º de marzo', { locale: 'es' }], + ['el 3 de marzo', { locale: 'es' }], + ['el proximo lunes', { locale: 'es' }], + ['lunes que viene', { locale: 'es' }], ] export const dateRangeRows = [] diff --git a/packages/lingo/tests/corpus/locale-fr-contract-v1.json b/packages/lingo/tests/corpus/locale-fr-contract-v1.json index c90f41f..6feae0e 100644 --- a/packages/lingo/tests/corpus/locale-fr-contract-v1.json +++ b/packages/lingo/tests/corpus/locale-fr-contract-v1.json @@ -638,6 +638,204 @@ "end": 11 }, "confidence": 0.9 + }, + "dix-sept kg": { + "input": "dix-sept kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 17, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 1 + }, + "dix-huit kg": { + "input": "dix-huit kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 18, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 1 + }, + "dix-neuf kg": { + "input": "dix-neuf kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 19, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 1 + }, + "mille cinq cents kg": { + "input": "mille cinq cents kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 1500, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 19 + }, + "confidence": 1 + }, + "deux cent cinquante mille kg": { + "input": "deux cent cinquante mille kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 250000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 28 + }, + "confidence": 1 + }, + "cent vingt kg": { + "input": "cent vingt kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 120, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 1 + }, + "un million de kg": { + "input": "un million de kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 1000000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 1 + }, + "deux milliards de kg": { + "input": "deux milliards de kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 2000000000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 20 + }, + "confidence": 1 + }, + "un billion de kg": { + "input": "un billion de kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 1000000000000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 1 + }, + "mille millions de kg": { + "input": "mille millions de kg", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "mass", + "base": 1000000000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 20 + }, + "confidence": 1 + }, + "20 euros": { + "input": "20 euros", + "opts": { + "locale": "fr" + }, + "type": "quantity", + "kind": "currency", + "base": 20, + "unit": "EUR", + "issues": [], + "span": { + "start": 0, + "end": 8 + }, + "confidence": 1 + }, + "entre mille et deux mille kg": { + "input": "entre mille et deux mille kg", + "opts": { + "locale": "fr" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 1000, + "max": 2000 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 6, + "end": 28 + }, + "confidence": 1 } }, "date": { @@ -939,6 +1137,75 @@ "end": 15 }, "confidence": 1 + }, + "le 1er mars": { + "input": "le 1er mars", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2027, + "month": 3, + "day": 1, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 1 + }, + "le 3 mars": { + "input": "le 3 mars", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2027, + "month": 3, + "day": 3, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 9 + }, + "confidence": 1 + }, + "lundi prochain": { + "input": "lundi prochain", + "opts": { + "locale": "fr" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 6, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 14 + }, + "confidence": 1 } }, "dateRange": {} diff --git a/packages/lingo/tests/corpus/locale-fr-source.mjs b/packages/lingo/tests/corpus/locale-fr-source.mjs index 2dba26f..f9baef7 100644 --- a/packages/lingo/tests/corpus/locale-fr-source.mjs +++ b/packages/lingo/tests/corpus/locale-fr-source.mjs @@ -50,6 +50,23 @@ export const breadthRows = [ ['une centaine de kg', { locale: 'fr' }], ['un millier de kg', { locale: 'fr' }], ['quelques kg', { locale: 'fr' }], + // Teens: the hyphen must not read as a range separator + ['dix-sept kg', { locale: 'fr' }], + ['dix-huit kg', { locale: 'fr' }], + ['dix-neuf kg', { locale: 'fr' }], + // Hundreds attach to the group in front of them, not the running total + ['mille cinq cents kg', { locale: 'fr' }], + ['deux cent cinquante mille kg', { locale: 'fr' }], + ['cent vingt kg', { locale: 'fr' }], + // Long scale: milliard is 10^9, billion is 10^12 + ['un million de kg', { locale: 'fr' }], + ['deux milliards de kg', { locale: 'fr' }], + ['un billion de kg', { locale: 'fr' }], + ['mille millions de kg', { locale: 'fr' }], + // Currency + ['20 euros', { locale: 'fr' }], + // The range and-word wins over number composition on both sides + ['entre mille et deux mille kg', { locale: 'fr' }], ] export const dateRows = [ @@ -71,6 +88,10 @@ export const dateRows = [ // localized weekday offsets ['lundi en huit', { locale: 'fr' }], ['mardi en quinze', { locale: 'fr' }], + // Ordinal day-of-month ("1er" only; other days are cardinal in French) + ['le 1er mars', { locale: 'fr' }], + ['le 3 mars', { locale: 'fr' }], + ['lundi prochain', { locale: 'fr' }], ] export const dateRangeRows = [] diff --git a/packages/lingo/tests/corpus/locale-ja-contract-v1.json b/packages/lingo/tests/corpus/locale-ja-contract-v1.json index c3f040d..f574586 100644 --- a/packages/lingo/tests/corpus/locale-ja-contract-v1.json +++ b/packages/lingo/tests/corpus/locale-ja-contract-v1.json @@ -314,6 +314,202 @@ "end": 1 }, "confidence": 1 + }, + "1000円": { + "input": "1000円", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "currency", + "base": 1000, + "unit": "JPY", + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "¥1000": { + "input": "¥1000", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "currency", + "base": 1000, + "unit": "JPY", + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "5キロ未満": { + "input": "5キロ未満", + "opts": { + "locale": "ja" + }, + "type": "range", + "kind": "mass", + "base": { + "min": null, + "max": 5 + }, + "unit": { + "min": null, + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "5キロ以上": { + "input": "5キロ以上", + "opts": { + "locale": "ja" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": null + }, + "unit": { + "min": "kg", + "max": null + }, + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "5キロ以下": { + "input": "5キロ以下", + "opts": { + "locale": "ja" + }, + "type": "range", + "kind": "mass", + "base": { + "min": null, + "max": 5 + }, + "unit": { + "min": null, + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "5キロ以内": { + "input": "5キロ以内", + "opts": { + "locale": "ja" + }, + "type": "range", + "kind": "mass", + "base": { + "min": null, + "max": 5 + }, + "unit": { + "min": null, + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "5キロ超": { + "input": "5キロ超", + "opts": { + "locale": "ja" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": null + }, + "unit": { + "min": "kg", + "max": null + }, + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "最低5キロ": { + "input": "最低5キロ", + "opts": { + "locale": "ja" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": null + }, + "unit": { + "min": "kg", + "max": null + }, + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "5キロほど": { + "input": "5キロほど", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 0.9 + }, + "5キロ前後": { + "input": "5キロ前後", + "opts": { + "locale": "ja" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 0.9 } }, "date": { @@ -684,6 +880,217 @@ "end": 2 }, "confidence": 1 + }, + "2026年3月5日": { + "input": "2026年3月5日", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 3, + "day": 5, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 9 + }, + "confidence": 1 + }, + "3月5日": { + "input": "3月5日", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 3, + "day": 5, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "水曜日": { + "input": "水曜日", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 8, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [ + "WEEKDAY_ASSUMED_NEXT" + ], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 0.9 + }, + "水曜": { + "input": "水曜", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 8, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [ + "WEEKDAY_ASSUMED_NEXT" + ], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 0.9 + }, + "来週の月曜日": { + "input": "来週の月曜日", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 6, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "午後3時": { + "input": "午後3時", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 3, + "hour": 15, + "minute": 0, + "second": 0 + }, + "unit": "hour", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "午前9時半": { + "input": "午前9時半", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 9, + "minute": 30, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "3時15分": { + "input": "3時15分", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 3, + "minute": 15, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "明日午後3時": { + "input": "明日午後3時", + "opts": { + "locale": "ja" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 15, + "minute": 0, + "second": 0 + }, + "unit": "hour", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 } }, "dateRange": {} diff --git a/packages/lingo/tests/corpus/locale-ja-source.mjs b/packages/lingo/tests/corpus/locale-ja-source.mjs index 620fa79..fc5eef8 100644 --- a/packages/lingo/tests/corpus/locale-ja-source.mjs +++ b/packages/lingo/tests/corpus/locale-ja-source.mjs @@ -19,6 +19,20 @@ export const breadthRows = [ ['約5キロ', { locale: 'ja' }], ['5キロ ほど', { locale: 'ja' }], ['三個', { locale: 'ja' }], + // Currency: the pack default disambiguates ¥ from the yuan + ['1000円', { locale: 'ja' }], + ['¥1000', { locale: 'ja' }], + // Postpositional bounds + ['5キロ未満', { locale: 'ja' }], + ['5キロ以上', { locale: 'ja' }], + ['5キロ以下', { locale: 'ja' }], + ['5キロ以内', { locale: 'ja' }], + ['5キロ超', { locale: 'ja' }], + // Prepositional bound + ['最低5キロ', { locale: 'ja' }], + // Trailing approximation, unspaced + ['5キロほど', { locale: 'ja' }], + ['5キロ前後', { locale: 'ja' }], ] export const dateRows = [ @@ -38,6 +52,19 @@ export const dateRows = [ ['先々月', { locale: 'ja' }], ['週末', { locale: 'ja' }], ['月末', { locale: 'ja' }], + // Numeric dates written with 年/月/日 suffixes + ['2026年3月5日', { locale: 'ja' }], + ['3月5日', { locale: 'ja' }], + // Weekdays + ['水曜日', { locale: 'ja' }], + ['水曜', { locale: 'ja' }], + ['来週の月曜日', { locale: 'ja' }], + // Clock with 時/分 and day periods + ['午後3時', { locale: 'ja' }], + ['午前9時半', { locale: 'ja' }], + ['3時15分', { locale: 'ja' }], + // Date + time with no separating space + ['明日午後3時', { locale: 'ja' }], ] export const dateRangeRows = [] diff --git a/packages/lingo/tests/corpus/locale-pt-contract-v1.json b/packages/lingo/tests/corpus/locale-pt-contract-v1.json index 7c2f8df..b659c70 100644 --- a/packages/lingo/tests/corpus/locale-pt-contract-v1.json +++ b/packages/lingo/tests/corpus/locale-pt-contract-v1.json @@ -726,6 +726,140 @@ "end": 12 }, "confidence": 1 + }, + "cento e vinte kg": { + "input": "cento e vinte kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 120, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 16 + }, + "confidence": 1 + }, + "mil e quinhentos kg": { + "input": "mil e quinhentos kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 1500, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 19 + }, + "confidence": 1 + }, + "dois mil e quinhentos kg": { + "input": "dois mil e quinhentos kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 2500, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 24 + }, + "confidence": 1 + }, + "um milhao de kg": { + "input": "um milhao de kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 1000000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + }, + "um bilhao de kg": { + "input": "um bilhao de kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 1000000000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 15 + }, + "confidence": 1 + }, + "mil milhoes de kg": { + "input": "mil milhoes de kg", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "mass", + "base": 1000000000, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 17 + }, + "confidence": 1 + }, + "20 reais": { + "input": "20 reais", + "opts": { + "locale": "pt" + }, + "type": "quantity", + "kind": "currency", + "base": 20, + "unit": "BRL", + "issues": [], + "span": { + "start": 0, + "end": 8 + }, + "confidence": 1 + }, + "entre mil e dois mil kg": { + "input": "entre mil e dois mil kg", + "opts": { + "locale": "pt" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 1000, + "max": 2000 + }, + "unit": { + "min": "kg", + "max": "kg" + }, + "issues": [], + "span": { + "start": 6, + "end": 23 + }, + "confidence": 1 } }, "date": { @@ -1096,6 +1230,173 @@ "end": 14 }, "confidence": 1 + }, + "segunda-feira": { + "input": "segunda-feira", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 6, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [ + "WEEKDAY_ASSUMED_NEXT" + ], + "span": { + "start": 0, + "end": 13 + }, + "confidence": 0.9 + }, + "terca-feira": { + "input": "terca-feira", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 7, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [ + "WEEKDAY_ASSUMED_NEXT" + ], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 0.9 + }, + "sexta-feira": { + "input": "sexta-feira", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 3, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [ + "WEEKDAY_ASSUMED_NEXT" + ], + "span": { + "start": 0, + "end": 11 + }, + "confidence": 0.9 + }, + "proxima segunda-feira": { + "input": "proxima segunda-feira", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 6, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 21 + }, + "confidence": 1 + }, + "na proxima segunda-feira": { + "input": "na proxima segunda-feira", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 6, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 24 + }, + "confidence": 1 + }, + "segunda-feira que vem": { + "input": "segunda-feira que vem", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 6, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 21 + }, + "confidence": 1 + }, + "3 de marco": { + "input": "3 de marco", + "opts": { + "locale": "pt" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2027, + "month": 3, + "day": 3, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 10 + }, + "confidence": 1 } }, "dateRange": {} diff --git a/packages/lingo/tests/corpus/locale-pt-source.mjs b/packages/lingo/tests/corpus/locale-pt-source.mjs index 7b9d62f..13dbaa8 100644 --- a/packages/lingo/tests/corpus/locale-pt-source.mjs +++ b/packages/lingo/tests/corpus/locale-pt-source.mjs @@ -56,6 +56,18 @@ export const breadthRows = [ ['dezanove kg', { locale: 'pt' }], ['dezesseis kg', { locale: 'pt' }], ['dezasseis kg', { locale: 'pt' }], + // "cento" is a bare scale; the and-word links it to the remainder + ['cento e vinte kg', { locale: 'pt' }], + ['mil e quinhentos kg', { locale: 'pt' }], + ['dois mil e quinhentos kg', { locale: 'pt' }], + // Brazilian short scale + ['um milhao de kg', { locale: 'pt' }], + ['um bilhao de kg', { locale: 'pt' }], + ['mil milhoes de kg', { locale: 'pt' }], + // Currency + ['20 reais', { locale: 'pt' }], + // The range and-word wins over number composition on both sides + ['entre mil e dois mil kg', { locale: 'pt' }], ] export const dateRows = [ @@ -79,6 +91,14 @@ export const dateRows = [ ['meio de julho', { locale: 'pt' }], ['fim de julho', { locale: 'pt' }], ['final de julho', { locale: 'pt' }], + // -feira weekdays, bare and with a locative contraction + ['segunda-feira', { locale: 'pt' }], + ['terca-feira', { locale: 'pt' }], + ['sexta-feira', { locale: 'pt' }], + ['proxima segunda-feira', { locale: 'pt' }], + ['na proxima segunda-feira', { locale: 'pt' }], + ['segunda-feira que vem', { locale: 'pt' }], + ['3 de marco', { locale: 'pt' }], ] export const dateRangeRows = [] diff --git a/packages/lingo/tests/corpus/locale-zh-contract-v1.json b/packages/lingo/tests/corpus/locale-zh-contract-v1.json index 08ba954..3a86c35 100644 --- a/packages/lingo/tests/corpus/locale-zh-contract-v1.json +++ b/packages/lingo/tests/corpus/locale-zh-contract-v1.json @@ -292,6 +292,300 @@ "end": 3 }, "confidence": 1 + }, + "100元": { + "input": "100元", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "currency", + "base": 100, + "unit": "CNY", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "一百元": { + "input": "一百元", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "currency", + "base": 100, + "unit": "CNY", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "50块": { + "input": "50块", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "currency", + "base": 50, + "unit": "CNY", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "¥100": { + "input": "¥100", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "currency", + "base": 100, + "unit": "CNY", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "5公斤以上": { + "input": "5公斤以上", + "opts": { + "locale": "zh" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": null + }, + "unit": { + "min": "kg", + "max": null + }, + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "5公斤以下": { + "input": "5公斤以下", + "opts": { + "locale": "zh" + }, + "type": "range", + "kind": "mass", + "base": { + "min": null, + "max": 5 + }, + "unit": { + "min": null, + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "5公斤以内": { + "input": "5公斤以内", + "opts": { + "locale": "zh" + }, + "type": "range", + "kind": "mass", + "base": { + "min": null, + "max": 5 + }, + "unit": { + "min": null, + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "大于5公斤": { + "input": "大于5公斤", + "opts": { + "locale": "zh" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": null + }, + "unit": { + "min": "kg", + "max": null + }, + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "小于5公斤": { + "input": "小于5公斤", + "opts": { + "locale": "zh" + }, + "type": "range", + "kind": "mass", + "base": { + "min": null, + "max": 5 + }, + "unit": { + "min": null, + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "不超过5公斤": { + "input": "不超过5公斤", + "opts": { + "locale": "zh" + }, + "type": "range", + "kind": "mass", + "base": { + "min": null, + "max": 5 + }, + "unit": { + "min": null, + "max": "kg" + }, + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "至少5公斤": { + "input": "至少5公斤", + "opts": { + "locale": "zh" + }, + "type": "range", + "kind": "mass", + "base": { + "min": 5, + "max": null + }, + "unit": { + "min": "kg", + "max": null + }, + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "五公斤左右": { + "input": "五公斤左右", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 0.9 + }, + "5公斤左右": { + "input": "5公斤左右", + "opts": { + "locale": "zh" + }, + "type": "quantity", + "kind": "mass", + "base": 5, + "unit": "kg", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 0.9 + }, + "三至五天": { + "input": "三至五天", + "opts": { + "locale": "zh" + }, + "type": "range", + "kind": "duration", + "base": { + "min": 259200, + "max": 432000 + }, + "unit": { + "min": "d", + "max": "d" + }, + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "三到五天": { + "input": "三到五天", + "opts": { + "locale": "zh" + }, + "type": "range", + "kind": "duration", + "base": { + "min": 259200, + "max": 432000 + }, + "unit": { + "min": "d", + "max": "d" + }, + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 } }, "date": { @@ -593,6 +887,309 @@ "end": 2 }, "confidence": 1 + }, + "2026年3月5日": { + "input": "2026年3月5日", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 3, + "day": 5, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 9 + }, + "confidence": 1 + }, + "3月5日": { + "input": "3月5日", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 3, + "day": 5, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "二〇二六年三月五日": { + "input": "二〇二六年三月五日", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 3, + "day": 5, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 9 + }, + "confidence": 1 + }, + "星期三": { + "input": "星期三", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 8, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [ + "WEEKDAY_ASSUMED_NEXT" + ], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 0.9 + }, + "周三": { + "input": "周三", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 8, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [ + "WEEKDAY_ASSUMED_NEXT" + ], + "span": { + "start": 0, + "end": 2 + }, + "confidence": 0.9 + }, + "下周一": { + "input": "下周一", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 6, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "上周五": { + "input": "上周五", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 6, + "day": 26, + "hour": 0, + "minute": 0, + "second": 0 + }, + "unit": "day", + "issues": [], + "span": { + "start": 0, + "end": 3 + }, + "confidence": 1 + }, + "下午3点": { + "input": "下午3点", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 3, + "hour": 15, + "minute": 0, + "second": 0 + }, + "unit": "hour", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "上午9点半": { + "input": "上午9点半", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 9, + "minute": 30, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 5 + }, + "confidence": 1 + }, + "3点一刻": { + "input": "3点一刻", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 3, + "minute": 15, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 4 + }, + "confidence": 1 + }, + "晚上8点30分": { + "input": "晚上8点30分", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 3, + "hour": 20, + "minute": 30, + "second": 0 + }, + "unit": "minute", + "issues": [], + "span": { + "start": 0, + "end": 7 + }, + "confidence": 1 + }, + "明天下午3点": { + "input": "明天下午3点", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 4, + "hour": 15, + "minute": 0, + "second": 0 + }, + "unit": "hour", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 + }, + "昨天上午9点": { + "input": "昨天上午9点", + "opts": { + "locale": "zh" + }, + "type": "date", + "kind": "date", + "base": { + "year": 2026, + "month": 7, + "day": 2, + "hour": 9, + "minute": 0, + "second": 0 + }, + "unit": "hour", + "issues": [], + "span": { + "start": 0, + "end": 6 + }, + "confidence": 1 } }, "dateRange": {} diff --git a/packages/lingo/tests/corpus/locale-zh-source.mjs b/packages/lingo/tests/corpus/locale-zh-source.mjs index 0d0275f..5f85d7c 100644 --- a/packages/lingo/tests/corpus/locale-zh-source.mjs +++ b/packages/lingo/tests/corpus/locale-zh-source.mjs @@ -18,6 +18,26 @@ export const breadthRows = [ ['差不多5公斤', { locale: 'zh' }], ['三 或 四 公斤', { locale: 'zh' }], ['七八天', { locale: 'zh' }], + // Currency: the pack default disambiguates ¥ from the yen + ['100元', { locale: 'zh' }], + ['一百元', { locale: 'zh' }], + ['50块', { locale: 'zh' }], + ['¥100', { locale: 'zh' }], + // Postpositional bounds + ['5公斤以上', { locale: 'zh' }], + ['5公斤以下', { locale: 'zh' }], + ['5公斤以内', { locale: 'zh' }], + // Prepositional bounds + ['大于5公斤', { locale: 'zh' }], + ['小于5公斤', { locale: 'zh' }], + ['不超过5公斤', { locale: 'zh' }], + ['至少5公斤', { locale: 'zh' }], + // Trailing approximation + ['五公斤左右', { locale: 'zh' }], + ['5公斤左右', { locale: 'zh' }], + // Range separators + ['三至五天', { locale: 'zh' }], + ['三到五天', { locale: 'zh' }], ] export const dateRows = [ @@ -34,6 +54,23 @@ export const dateRows = [ ['年底', { locale: 'zh' }], ['年初', { locale: 'zh' }], ['周末', { locale: 'zh' }], + // Numeric dates written with 年/月/日 suffixes + ['2026年3月5日', { locale: 'zh' }], + ['3月5日', { locale: 'zh' }], + ['二〇二六年三月五日', { locale: 'zh' }], + // Weekdays + ['星期三', { locale: 'zh' }], + ['周三', { locale: 'zh' }], + ['下周一', { locale: 'zh' }], + ['上周五', { locale: 'zh' }], + // Clock with 点 and day periods + ['下午3点', { locale: 'zh' }], + ['上午9点半', { locale: 'zh' }], + ['3点一刻', { locale: 'zh' }], + ['晚上8点30分', { locale: 'zh' }], + // Date + time with no separating space + ['明天下午3点', { locale: 'zh' }], + ['昨天上午9点', { locale: 'zh' }], ] export const dateRangeRows = [] diff --git a/packages/lingo/tests/corpus/source.mjs b/packages/lingo/tests/corpus/source.mjs index 07a391c..eeff959 100644 --- a/packages/lingo/tests/corpus/source.mjs +++ b/packages/lingo/tests/corpus/source.mjs @@ -190,6 +190,13 @@ export const breadthRows = [ ['one hundred and five kg'], ['two point five six kg'], ['three point one four kg'], + // "between A and B" where both sides are spelled scale words: the and-word + // separates the range instead of composing a single number. + ['between one thousand and two thousand meters'], + ['between two hundred and five hundred meters'], + ['between five and a half and ten kg'], + ['nineteen hundred kg'], + ['two hundred thousand kg'], [''], [' '], ['banana'], diff --git a/packages/lingo/tests/locales/cjk.test.ts b/packages/lingo/tests/locales/cjk.test.ts index b803a11..403933b 100644 --- a/packages/lingo/tests/locales/cjk.test.ts +++ b/packages/lingo/tests/locales/cjk.test.ts @@ -387,3 +387,139 @@ describe('CJK locale packs', () => { } }) }) + +// ─── Suffix-delimited dates and clocks ────────────────────────────────────── + +describe('CJK suffix dates and clocks', () => { + const dates = [ + ['zh', '2026年3月5日', new Date(2026, 2, 5)], + ['zh', '3月5日', new Date(2026, 2, 5)], + ['zh', '二〇二六年三月五日', new Date(2026, 2, 5)], + ['ja', '2026年3月5日', new Date(2026, 2, 5)], + ['ja', '3月5日', new Date(2026, 2, 5)], + ] as const + + it.each(dates)('%s %s reads the 年/月/日 suffixes', (locale, input, expected) => { + const result = parseDate(input, { now: NOW, locale, localePacks: [zh, ja] }) + expect(result.ok, `${input}: ${JSON.stringify(result.ok ? '' : result.issues)}`).toBe(true) + if (result.ok) { + expect(result.date).toEqual(expected) + expect(result.grain).toBe('day') + } + }) + + const clocks = [ + ['zh', '下午3点', new Date(2026, 6, 8, 15)], + ['zh', '上午9点半', new Date(2026, 6, 9, 9, 30)], + ['zh', '3点一刻', new Date(2026, 6, 9, 3, 15)], + ['zh', '晚上8点30分', new Date(2026, 6, 8, 20, 30)], + ['ja', '午後3時', new Date(2026, 6, 8, 15)], + ['ja', '午前9時半', new Date(2026, 6, 9, 9, 30)], + ['ja', '3時15分', new Date(2026, 6, 9, 3, 15)], + ] as const + + it.each(clocks)('%s %s reads the day period and clock suffix', (locale, input, expected) => { + const result = parseDate(input, { now: NOW, locale, localePacks: [zh, ja] }) + expect(result.ok, `${input}: ${JSON.stringify(result.ok ? '' : result.issues)}`).toBe(true) + if (result.ok) { + expect(result.date).toEqual(expected) + } + }) + + // Date and time run together with no separating space. + const unspaced = [ + ['zh', '明天下午3点', new Date(2026, 6, 9, 15)], + ['zh', '昨天上午9点', new Date(2026, 6, 7, 9)], + ['ja', '明日午後3時', new Date(2026, 6, 9, 15)], + ] as const + + it.each(unspaced)('%s %s splits date from time without a space', (locale, input, expected) => { + const result = parseDate(input, { now: NOW, locale, localePacks: [zh, ja] }) + expect(result.ok, `${input}: ${JSON.stringify(result.ok ? '' : result.issues)}`).toBe(true) + if (result.ok) { + expect(result.date).toEqual(expected) + } + }) + + const weekdays = [ + ['zh', '星期三', new Date(2026, 6, 8)], + ['zh', '周三', new Date(2026, 6, 8)], + ['ja', '水曜日', new Date(2026, 6, 8)], + ['ja', '水曜', new Date(2026, 6, 8)], + ] as const + + it.each(weekdays)('%s %s resolves the weekday', (locale, input, expected) => { + const result = parseDate(input, { now: NOW, locale, localePacks: [zh, ja] }) + expect(result.ok, `${input}: ${JSON.stringify(result.ok ? '' : result.issues)}`).toBe(true) + if (result.ok) { + expect(result.date).toEqual(expected) + } + }) +}) + +// ─── Postpositional bounds ────────────────────────────────────────────────── + +describe('CJK postpositional bounds', () => { + const lingo = createLingo({ locales: [zh, ja] }) + + const bounds = [ + ['zh', '5公斤以上', 5, null], + ['zh', '5公斤以下', null, 5], + ['zh', '5公斤以内', null, 5], + ['zh', '大于5公斤', 5, null], + ['zh', '小于5公斤', null, 5], + ['zh', '不超过5公斤', null, 5], + ['zh', '至少5公斤', 5, null], + ['ja', '5キロ未満', null, 5], + ['ja', '5キロ以上', 5, null], + ['ja', '5キロ以下', null, 5], + ['ja', '5キロ以内', null, 5], + ['ja', '5キロ超', 5, null], + ['ja', '最低5キロ', 5, null], + ] as const + + it.each(bounds)('%s %s bounds the range', (locale, input, min, max) => { + const result = lingo.parseRange(input, { locale }) + expect(result.ok, `${input}: ${JSON.stringify(result.ok ? '' : result.issues)}`).toBe(true) + if (!result.ok) { + return + } + expect(result.range.min()?.value ?? null).toBe(min) + expect(result.range.max()?.value ?? null).toBe(max) + }) + + // GUARD: 間 is a range word but 時間 is a unit — the unit must stay whole. + it('keeps 一時間半 as 1.5 hours, not a range across 間', () => { + const result = lingo.parseQuantity('一時間半', { locale: 'ja' }) + expect(result.ok, JSON.stringify(result.ok ? '' : result.issues)).toBe(true) + if (result.ok) { + expect(result.quantity.unit).toBe('h') + expect(result.quantity.value).toBe(1.5) + } + }) +}) + +// ─── Currency defaults ────────────────────────────────────────────────────── + +describe('CJK currency', () => { + const lingo = createLingo({ locales: [zh, ja] }) + + const money = [ + ['zh', '100元', 100, 'CNY'], + ['zh', '一百元', 100, 'CNY'], + ['zh', '50块', 50, 'CNY'], + // ¥ is shared between the two currencies; the pack default decides. + ['zh', '¥100', 100, 'CNY'], + ['ja', '1000円', 1000, 'JPY'], + ['ja', '¥1000', 1000, 'JPY'], + ] as const + + it.each(money)('%s %s reads as %d %s', (locale, input, value, unit) => { + const result = lingo.parseQuantity(input, { locale }) + expect(result.ok, `${input}: ${JSON.stringify(result.ok ? '' : result.issues)}`).toBe(true) + if (result.ok) { + expect(result.quantity.value).toBe(value) + expect(result.quantity.unit).toBe(unit) + } + }) +}) diff --git a/plans/031-locale-packs.md b/plans/031-locale-packs.md index 3810056..f403d92 100644 --- a/plans/031-locale-packs.md +++ b/plans/031-locale-packs.md @@ -105,6 +105,18 @@ Published packs: dates like `12 de julio de 2026`, and compact CJK offset units/suffixes. The date parser reads those tables through `LanguageProfile.date`; language strings do not live in the core date grammar. +9. Suffix-delimited date/clock vocabulary (D70): `DateVocabPack` gained + `numericDateSuffixes` (年/月/日), `clockSuffix` (点/時/分/秒 plus minute words + such as 半/一刻), `dayPeriods` (上午/下午, 午前/午後), and `ordinalSuffixes` + (Romance `1er`/`1º`). `date/suffix.ts` consumes them; `date/numeral.ts` reads + locale numerals (positional `十`-tens plus digit-run years like `二〇二六`). + A pack that declares none of these fields never enters those paths. +10. Glued-script tokenization and postpositional bounds (D70): the tokenizer + splits word tokens at the profile's non-Latin grammar vocabulary so glued + grammar is visible to the parser, suppressed inside unit aliases (`一時間半` + stays one hour-and-a-half, not a range at `間`). `GrammarBoundPhrase.suffix` + marks comparators that follow the quantity (`以上`, `未満`), read by + `parseTrailingBound` in the quantity and range paths. ## Date locale policy @@ -130,7 +142,9 @@ Published packs: - Translating the built-in English unit tables. - Shipping all locale packs through the default `.` entry. - Localized `humanizeDate()` / `humanizeDuration()` output in Phase 0. -- Full CJK segmentation or compound numeral grammar. +- Full CJK segmentation or compound numeral grammar. (D70 splits word tokens at + pack-declared grammar vocabulary; that is targeted grammar-boundary cutting, + not general segmentation — there is still no dictionary or statistical model.) - Changing serialized result JSON in the v3 contract. ## Acceptance diff --git a/plans/033-locale-idiom-coverage.md b/plans/033-locale-idiom-coverage.md index a195e97..bb2d37a 100644 --- a/plans/033-locale-idiom-coverage.md +++ b/plans/033-locale-idiom-coverage.md @@ -3,7 +3,7 @@ id: 033 title: Locale idiom coverage status: in-progress created: 2026-07-09 -updated: 2026-07-09 +updated: 2026-07-27 --- # Locale idiom coverage — parse how people actually write and speak @@ -71,6 +71,40 @@ localized humanize output (D66 deferral stands), counters/classifiers beyond filler handling, height/weight elliptical speech ("uno ochenta"), currency slang ("a grand", "5 lucas"). +## Wave 2 (D70, 2026-07-27) + +Wave 1 shipped CJK *numbers* but not CJK *calendars*, and shipped prefix bounds +but not postpositional ones. Wave 2 closes both, again through pack data +consumed by language-neutral engine seams: + +- **Glued-token grammar splitting** — scripts without word spaces write grammar + and content as one token, so the tokenizer cuts word tokens at the profile's + non-Latin grammar vocabulary (range separators, bound phrases, trailing + approximants) before the grammar runs. The cut is suppressed inside a unit + alias, so `一時間半` never splits at the range word `間`. This retires the + wave-1 deferral for no-space trailing approximants (`五公斤左右`). +- **Postpositional bounds** — `GrammarBoundPhrase.suffix` marks comparators that + follow the quantity (`以上`, `以下`, `未満`, `以内`, `超`), consumed by + `parseTrailingBound` in both the quantity and range paths. Previously + `5キロ未満` silently parsed as a bare `5 kg`. +- **Suffix-delimited dates and clocks** — new `DateNumericSuffixVocab` + (年/月/日), `DateClockSuffixVocab` (点/時/分/秒 plus minute words 半/一刻), and + `DateDayPeriod` (上午/下午, 午前/午後) fields, consumed by `date/suffix.ts`; + `date/numeral.ts` reads locale numerals including digit-by-digit years + (`二〇二六年`). Unspaced date+time compounds (`明天下午3点`) split at day-period + and clock-suffix anchors. This retires the wave-1 CJK spoken-clock deferral. +- **Localized calendars** — weekday and month vocabulary for zh/ja, `ordinalSuffixes` + for Romance day-of-month (`le 1er mars`), Portuguese locative date fillers. +- **Number-word arithmetic corrections** — three silent wrong answers fixed + (hundreds binding, scale chaining, and-word after bare scales) plus `noAnd` + threading so `between one thousand and two thousand` reads as a range. See D70. +- **Per-locale benchmarks and profile memoization** — resolved profiles and + detection scans are cached per pack set; `bun run bench` covers each locale. + +Wave-3 candidates are in `plans/backlog.md` under "Wave-2 idiom deferrals" +(Japanese era years, CJK suffix-date ranges, `3月5日(木)` weekday agreement, +positional CJK numerals in dates). + ## Acceptance - Per-locale corpus gates run in `bun run check`; every wave-1 idiom class has diff --git a/plans/backlog.md b/plans/backlog.md index c26be5b..f75dd31 100644 --- a/plans/backlog.md +++ b/plans/backlog.md @@ -285,11 +285,11 @@ full context. ## Wave-1 idiom deferrals (2026-07-09, D69) -- **No-space CJK trailing approx** — `五公斤左右` / `五キロぐらい` still fail: - the suffix particle merges into the unit token; the unit matcher needs - approx-suffix stripping for CJK word tokens (spaced forms work today). -- **CJK spoken clock** — `午後3時半` / `三点一刻` need the D69 clock fields to - compose with the CJK tokenizer path; deferred from the date-grammar package. +Delivered by wave 2 (D70): no-space CJK trailing approx (`五公斤左右`, +`五キロぐらい`, `五公斤上下`) via profile-driven splitting of glued grammar +words, and the CJK spoken clock (`午後3時半`, `三点一刻`, `下午三点半`) via the +suffix clock grammar. Still open: + - **Unit-first implied-1 quantities** — `kilo y medio`, `hora y media`, `metro e meio`: unit-before-number with implied quantity 1; needs a quantity-grammar seam, was cut from D69 for size. @@ -298,3 +298,29 @@ full context. - **`个` classifier before duration units in ranges** — `三四个小时` composes adjacent-number range + classifier + unit; blocked on the classifier field (see counters/classifiers item above). + +## Wave-2 idiom deferrals (2026-07-27, D70) + +- **Japanese era years** — `令和8年3月5日` / `平成30年`: the numeric-suffix date + path reads `年` but not an era prefix. Needs an `eraNames` pack field mapping + era → Gregorian offset, and a policy for the transition year. `Intl` knows the + Japanese calendar, but using it would make date parsing environment-dependent + (D5-adjacent: determinism is the guarantee). +- **CJK numeric-suffix date ranges** — `3月5日から3月10日` / `3月5日~10日`: the + suffix date parser is single-date; a range needs the same + `から`/`~` separator handling the quantity range path has, plus elliptical + right sides (`10日` inheriting the month from the left). +- **Weekday + numeric-suffix date agreement** — `3月5日(木)` is the standard + written form on Japanese schedules; the parenthesized weekday is currently + trailing input. Cheap to accept, but it should *validate* agreement (warn when + the weekday contradicts the date) rather than silently ignore it, which needs + a new issue code. +- **Glued splitting for spaced scripts** — the D70 splitter deliberately only + runs on non-Latin vocabulary (`!/[a-z]/i`). A pack for a spaced script that + still glues grammar (Finnish/Turkish agglutination, German compounds) would + need a different filter and a much stronger alias guard. +- **`readLocaleNumber` positional coverage** — the CJK numeral reader handles + bare numerals, `十`-based tens, and digit runs (for years), but not + `百`/`千` positional composition (`三百五十日` as a day-of-month is nonsense, + yet `二千二十六年` is a legal year spelling). Unify with the CJK number + walker in `number/` instead of growing a second reader. diff --git a/wiki/architecture.md b/wiki/architecture.md index 6cefe70..15d72dd 100644 --- a/wiki/architecture.md +++ b/wiki/architecture.md @@ -27,7 +27,7 @@ input string |-------|--------|----------| | `.` (main) | `src/index.ts` | batteries-included: parse, convert, format, `createLingo`, extension points | | `./core` | `src/core/` + `src/parse/` + `src/number/` + `src/format/` | engine without bundled unit data — BYO registry, copy-free (D14) | -| `./date` | `src/date/` | NL dates, time-of-day, zones, ranges/slots, reversible humanize, ISO durations | +| `./date` | `src/date/` | NL dates, time-of-day, zones, ranges/slots, reversible humanize, ISO durations; `suffix.ts`/`numeral.ts` handle suffix-delimited CJK dates and clocks (D70) | | `./dom` | `src/dom/` | headless input controller | | `./element` | `src/element/` | `` form-associated custom element | | `./react` | `src/react/` | `useLingoInput` hook (`'use client'`) over the DOM controller | @@ -82,6 +82,19 @@ resolver + detector; the packs themselves live in `src/locales/`). (5K is kelvin, 70k is 70 000). - **Registry refs are liberal**: `.to('L')`, `convert(1,'gal','L')` resolve through aliases; `Quantity.unit` canonicalizes to the id ('l'). +- **Glued-grammar splitting is alias-guarded** (D70): scripts without word spaces + glue grammar to content (`5キロ未満`), so `prepare()` cuts word tokens at the + profile's non-Latin grammar vocabulary before parsing. The cut must be + suppressed *inside a unit alias* — `間` is a range word but `時間` is the hour + unit, so `一時間半` would otherwise split into a range. That is why + `splitGluedWords` takes an alias-length callback: tokenization has to consult + the unit matcher. If a CJK unit ever parses as a range, look here first. +- **`noAnd` is threaded, not global** (D70): inside `between A and B` the and-word + belongs to the range, so `range.ts` passes `noAnd` through `QtyFlags` → + `ValueCtx` → the number-word engine, where it suppresses the scale link + (`one thousand and two thousand`) and the tens+ones link. It deliberately does + NOT suppress the fraction tail, so `between five and a half and ten kg` reads + 5.5..10. Blanking `andWords` wholesale looks equivalent and breaks that case. - **No hidden clock**: `parseDate()` reads no wall clock; reference-dependent inputs without `now` fail with `NOW_REQUIRED` (D36). The `/ai` date fields' `requireNow: false` is the one explicit opt-in to the host clock, at field diff --git a/wiki/decisions.md b/wiki/decisions.md index 1e2db62..d358ad7 100644 --- a/wiki/decisions.md +++ b/wiki/decisions.md @@ -544,3 +544,54 @@ and `./core` D68 budgets hold. Deferred to wave 2 (backlog): no-space CJK trailing approx (五公斤左右 needs suffix-stripping in the unit matcher), CJK spoken clock (午後3時半), unit-first implied-1 (`hora y media`, `kilo y medio`), 三点五 CJK decimal composition, `uno ochenta` elliptical heights. + +**D70 · 2026-07-27 · Wave-2 locale depth: glued-script tokenization, suffix +date/clock grammar, and number-word arithmetic corrections.** Four mechanisms, +each a real trade-off: + +(1) **Profile-driven splitting of glued grammar words.** Scripts without word +spaces write grammar and content as one token (`5キロ未満`, `三至五天`), so the +tokenizer now cuts word tokens at the profile's non-Latin grammar vocabulary +(range separators, bound phrases, trailing approximants) before the grammar sees +them. The cut is *suppressed inside a unit alias* — `間` is a range word but +`時間` is the hour unit, so `一時間半` must not split. That guard needs the unit +matcher during tokenization, which is why `prepare()` passes an alias-length +callback into `splitGluedWords`. Rejected: teaching each grammar rule to match +mid-token (the same ambiguity re-litigated at a dozen call sites). + +(2) **Postpositional bounds are declared, not inferred.** `GrammarBoundPhrase` +gained `suffix?: boolean`, because `以上`/`未満` follow the quantity while +`大于`/`最低` precede it, and a phrase that reads as both would silently pick +one. Marked phrases are skipped by the prefix scanner and picked up by +`parseTrailingBound`. Before this, `5キロ未満` parsed as a bare `5 kg` — a +silent wrong answer, the worst failure mode for a validation library. + +(3) **Suffix-delimited dates and clocks** (`date/suffix.ts`, `date/numeral.ts`): +`2026年3月5日`, `3時15分`, `下午3点半`, plus unspaced date+time compounds +(`明天下午3点`) split at day-period and clock-suffix anchors rather than at every +offset. Years spelled digit-by-digit (`二〇二六年`) read as a digit run, since +positional reading would give 2+0+2+6. All of it is pack-data-gated: a pack that +declares no `numericDateSuffixes`/`clockSuffix` never enters the path. + +(4) **Number-word arithmetic was wrong in three ways**, all silent: hundreds +multiplied the whole running total instead of the 1..99 group in front of them +(`mille cinq cents` = 100500, not 1500); a banked smaller scale added instead of +multiplying (`mil millones` = 1,001,000, not 10^9); and the and-word linking a +scale to its remainder was not offered after bare scales (`cento e vinte` +failed). Fixing the third exposed the mirror bug — inside `between A and B` the +and-word belongs to the range, so the existing `noAnd` flag now threads from +`range.ts` through `ValueCtx` into the word engine. It suppresses the scale and +tens links but deliberately NOT the fraction tail, so `between five and a half +and ten kg` still reads 5.5..10. + +Budgets recalibrate once for the wave (D19 escalation honored, owner approved): +main entry 38.3 → 39.4, `./core` 25.6 → 26.8, `./date` standalone 41.0 → 43.3 +and marginal 14.4 → 15.7, `./ai` 17.4 → 18.6 (cascade only, no `/ai` code +changed), zh/ja packs 1.3/1.4 → 2.0 each, cjk marginal 2.0 → 3.0, all-locales +6.3 → 6.9. Every existing corpus contract row is unchanged — the whole wave is +additive (+96 locale rows, +5 English rows). Also landed: zh/ja default +currencies so the shared `¥` resolves per locale, Portuguese `cento` and +locative fillers (`na proxima segunda-feira`), and a per-locale benchmark suite +so multi-language throughput is tracked rather than assumed. Revisit if a pack +wants glued splitting for a *Latin* script — the current filter is deliberately +`!/[a-z]/i`, on the theory that spaced scripts already tokenize correctly.