diff --git a/.changeset/28993-locale-case-canonicalized.md b/.changeset/28993-locale-case-canonicalized.md new file mode 100644 index 00000000000..213d329a0db --- /dev/null +++ b/.changeset/28993-locale-case-canonicalized.md @@ -0,0 +1,24 @@ +--- +'@objectstack/spec': minor +--- + +feat(spec)!: a locale tag written in another case is corrected to its BCP-47 case — a translation saved as `zh-cn` reaches `zh-CN` sessions — and a translation bundle with two keys that fold to one tag is refused + +Clause-②: no (narrowing) +Metadata-protocol: narrowed + +A translation saved with `locale: 'zh-cn'` answered 200, and a `zh-CN` session kept rendering the stock strings. Case was accepted as written on the premise that the resolvers fold it, but a resolver answers one locale for a request, exact match first, so the stock `zh-CN` bundle won and the `zh-cn` entry was never merged into it. + +- **The parse corrects case.** `LocaleSchema` returns each tag in its BCP-47 case: the language subtag in lower case, a script in title case, a region in upper case, and every subtag after a singleton in lower case. `zh-cn` reads back as `zh-CN`, `zh-hans-cn` as `zh-Hans-CN`, `EN` as `en`. This reaches every reader of the schema: a translation item's `locale`, a `TranslationBundle` / `PlatformTranslationBundle` locale key, `i18n.defaultLocale` / `fallbackLocale` / `supportedLocales`, a seed dataset's `locale` scope and the seed loader's `locale`. A canonical tag is returned unchanged, and nothing that parsed before is refused, with the one exception below. +- **Stored rows are read back canonical.** The metadata save door stores the authored body, so a row saved as `zh-cn` stays `zh-cn` at rest. A new stored-row conversion, `translation-locale-tag-case-folded`, folds case right after `translation-locale-tag-canonicalized` rewrites `_` and padding, and the runtime-authored translation sync replays both before keying a row by its locale. A row saved as `zh-cn` is served to `zh-CN` sessions from the next sync on, with nothing to re-save. An artifact built by `@objectstack/spec` 18.2.0 or earlier is folded the same way. +- **Case only.** The fold never replaces a legacy or macrolanguage subtag the way `Intl.getCanonicalLocales` does: `iw` stays `iw`, `cmn-cn` becomes `cmn-CN`. That keeps a translation on the locale its author named, and gives the same answer in every runtime. +- **Upgrade path.** The ADR-0087 D3 semantic entry `translation-locale-case-canonicalized` carries both halves to `os migrate meta` and the upgrade guide. No key is added or removed, and the published JSON Schema of a locale is unchanged. +- **Exact matchers that read a folded locale against a row tagged as written.** The email-template floor warning compares `i18n.supportedLocales` with `emailTemplates[].locale`, and the messaging template ladder compares the default locale with `sys_notification_template.locale`. A row tagged `zh-cn` no longer matches a supported or default locale authored as `zh-cn`, which now reads `zh-CN`: re-tag the row in its canonical case. + + + +**BREAKING** for an author whose one translation bundle spells a locale twice, in two cases. + +- **Before.** `{ 'zh-cn': { messages: { save: '保存' } }, 'zh-CN': { messages: { cancel: '取消' } } }` parsed as a `TranslationBundle` (or `PlatformTranslationBundle`) with both keys kept, and the `zh-cn` strings never reached a `zh-CN` session. +- **Now.** Both keys fold to `zh-CN`, so keeping both would put two entries on one key and silently drop one set of strings. The parse refuses the bundle at the second key, naming both keys and the tag they fold to. Two separate bundles in `stack.translations`, one with `zh-cn` and one with `zh-CN`, are still accepted; the runtime merges them as one locale. +- **Fix.** FROM a bundle holding both spellings, `{ 'zh-cn': A, 'zh-CN': B }`, TO one canonical key holding both sets of strings, `{ 'zh-CN': A and B merged }`, merged by hand: where both set the same string, keep the one you mean. No bundle in this repository has such a pair. A stored row with such a pair and different strings under the two keys is left as stored by the conversion and reported as a TODO. diff --git a/content/docs/protocol/kernel/i18n-standard.mdx b/content/docs/protocol/kernel/i18n-standard.mdx index 8e3ae9744ac..eed34c7ecbb 100644 --- a/content/docs/protocol/kernel/i18n-standard.mdx +++ b/content/docs/protocol/kernel/i18n-standard.mdx @@ -245,7 +245,12 @@ Every locale code (a bundle key, an item's `locale`, `i18n.defaultLocale`, between subtags (`en`, `zh-CN`, `zh-Hans-CN`). `LocaleSchema` refuses anything else at parse — `zh_CN`, a padded value, a language name — and the 18.x conversion `translation-locale-tag-canonicalized` rewrites the `_` and padded -spellings in stored rows and older builds. Case is not judged: the resolvers fold it. +spellings in stored rows and older builds. Case is corrected, never refused: the +parse returns each tag in its BCP-47 case (`zh-cn` → `zh-CN`, `zh-hans-cn` → +`zh-Hans-CN`), and the conversion `translation-locale-tag-case-folded` folds a +stored row's locale and an artifact built by 18.2.0 or earlier, so a +translation saved as `zh-cn` is served to `zh-CN` sessions. A bundle holding two +keys that fold to one tag (`zh-cn` beside `zh-CN`) is refused naming both. ```typescript // One locale of a TranslationBundle (e.g. zh-CN) diff --git a/packages/spec/dropped-refinements.baseline.json b/packages/spec/dropped-refinements.baseline.json index 2669c9f239a..f370c5b3a0f 100644 --- a/packages/spec/dropped-refinements.baseline.json +++ b/packages/spec/dropped-refinements.baseline.json @@ -3,7 +3,7 @@ "measured": { "zod": "4.4.3", "publishedSchemasWithDroppedRefinements": 250, - "droppedRefinementSites": 1012, + "droppedRefinementSites": 1018, "refinementSitesThatDidProject": 369, "refinementSitesWithNoJsonFormToCompare": 0 }, @@ -129,6 +129,7 @@ "manifest.reports.element.runtimeFilter.lazy", "manifest.sharingRules.element.sharedWith", "manifest.skills.element.triggerConditions.element", + "manifest.translations.element", "manifest.translations.element.valueType.flows.valueType.errorMessage", "manifest.translations.element.valueType.flows.valueType.notices.valueType.message", "manifest.translations.element.valueType.flows.valueType.refusals.valueType.message", @@ -280,6 +281,7 @@ "data.options[1].manifest.reports.element.runtimeFilter.lazy", "data.options[1].manifest.sharingRules.element.sharedWith", "data.options[1].manifest.skills.element.triggerConditions.element", + "data.options[1].manifest.translations.element", "data.options[1].manifest.translations.element.valueType.flows.valueType.errorMessage", "data.options[1].manifest.translations.element.valueType.flows.valueType.successMessage", "data.options[1].manifest.views.element.form", @@ -422,6 +424,7 @@ "options[1].manifest.reports.element.runtimeFilter.lazy", "options[1].manifest.sharingRules.element.sharedWith", "options[1].manifest.skills.element.triggerConditions.element", + "options[1].manifest.translations.element", "options[1].manifest.translations.element.valueType.flows.valueType.errorMessage", "options[1].manifest.translations.element.valueType.flows.valueType.notices.valueType.message", "options[1].manifest.translations.element.valueType.flows.valueType.refusals.valueType.message", @@ -504,6 +507,7 @@ "data.packages.element.options[1].manifest.reports.element.runtimeFilter.lazy", "data.packages.element.options[1].manifest.sharingRules.element.sharedWith", "data.packages.element.options[1].manifest.skills.element.triggerConditions.element", + "data.packages.element.options[1].manifest.translations.element", "data.packages.element.options[1].manifest.translations.element.valueType.flows.valueType.errorMessage", "data.packages.element.options[1].manifest.translations.element.valueType.flows.valueType.successMessage", "data.packages.element.options[1].manifest.views.element.form", @@ -1463,12 +1467,13 @@ }, "system/PlatformTranslationBundle": { "sites": [ + "", "valueType.flows.valueType.errorMessage", + "valueType.flows.valueType.notices.valueType.message", "valueType.flows.valueType.refusals.valueType.message", "valueType.flows.valueType.screens.valueType.description", "valueType.flows.valueType.screens.valueType.title", - "valueType.flows.valueType.successMessage", - "valueType.flows.valueType.notices.valueType.message" + "valueType.flows.valueType.successMessage" ] }, "system/PlatformTranslationData": { @@ -1533,12 +1538,13 @@ }, "system/TranslationBundle": { "sites": [ + "", "valueType.flows.valueType.errorMessage", + "valueType.flows.valueType.notices.valueType.message", "valueType.flows.valueType.refusals.valueType.message", "valueType.flows.valueType.screens.valueType.description", "valueType.flows.valueType.screens.valueType.title", - "valueType.flows.valueType.successMessage", - "valueType.flows.valueType.notices.valueType.message" + "valueType.flows.valueType.successMessage" ] }, "system/TranslationData": { diff --git a/packages/spec/src/conversions/registry.ts b/packages/spec/src/conversions/registry.ts index a12912c7d80..b96317815b9 100644 --- a/packages/spec/src/conversions/registry.ts +++ b/packages/spec/src/conversions/registry.ts @@ -4130,8 +4130,10 @@ const elementFormRemoved: MetadataConversion = { * * **What it rewrites.** Only the two spellings that have exactly one tag they * can have meant: `_` used as the subtag separator, and whitespace around the - * value. Case is kept as written, because every resolver already folds it. - * The result must pass `LocaleSchema`; a value that still does not (a + * value. Case is kept as written here; its fold is the entry after this one, + * {@link translationLocaleTagCaseFolded}, retired a release later, because the + * premise that every resolver folds case was measured false (#28993). The + * result must pass `LocaleSchema`; a value that still does not (a * language name, `not a locale!!`) is left as stored and reported as a TODO, * because only its author knows which locale it was for. * @@ -4262,8 +4264,9 @@ const translationLocaleTagCanonicalized: MetadataConversion = { translations: [ // A stored item under an underscore spelling. { name: 'zh_cn', locale: 'zh_CN', messages: { commonSave: '保存' } }, - // Already a tag, in lower case: resolvers fold case, so it is left alone. - { name: 'pt_br', locale: 'pt-br', messages: { commonSave: 'Salvar' } }, + // Already a tag: left alone. (A tag in another case is the next + // entry's, `translation-locale-tag-case-folded`; fixtures stay disjoint.) + { name: 'pt_br', locale: 'pt-BR', messages: { commonSave: 'Salvar' } }, // Not a tag and no single tag it can have meant: a TODO, left as stored. { name: 'qa_bad_locale', locale: 'not a locale!!', messages: { commonSave: 'x' } }, // A bundle entry: one padded key, one underscore key, one tag. @@ -4281,7 +4284,7 @@ const translationLocaleTagCanonicalized: MetadataConversion = { after: { translations: [ { name: 'zh_cn', locale: 'zh-CN', messages: { commonSave: '保存' } }, - { name: 'pt_br', locale: 'pt-br', messages: { commonSave: 'Salvar' } }, + { name: 'pt_br', locale: 'pt-BR', messages: { commonSave: 'Salvar' } }, { name: 'qa_bad_locale', locale: 'not a locale!!', messages: { commonSave: 'x' } }, { 'zh-CN': { messages: { commonSave: '保存' } }, @@ -4299,6 +4302,181 @@ const translationLocaleTagCanonicalized: MetadataConversion = { }, }; +/** + * A locale tag written in a case other than BCP-47's becomes the same tag in + * its canonical case (protocol 18, #28993): `zh-cn` → `zh-CN`, `zh-hans-cn` → + * `zh-Hans-CN`, `EN` → `en`. + * + * The case half of {@link translationLocaleTagCanonicalized}, kept as an entry + * of its own because it retired a release later. That entry left case as + * written, on the premise that every resolver folds it. The premise was + * measured false: a resolver answers one locale for a request, exact match + * first, so a `zh-cn` translation beside the stock `zh-CN` bundle was never + * merged into it, and a `zh-CN` session never saw its strings. `LocaleSchema` + * now folds case on parse, so a live author is corrected there. The metadata + * save door stores the authored body rather than the parse, so this stored-row + * pass is what carries the fold to a row saved in lower case, and to an + * artifact built by 18.2.0 or earlier. + * + * **What it rewrites.** A value that is already tag-shaped and differs from + * its `LocaleSchema` parse only in case. The fold is case only: the language + * subtag lower case, a 4-letter script title case, a 2-letter region upper + * case, every subtag after a singleton lower case. It never replaces a legacy + * subtag (`iw` stays `iw`). A value that is not tag-shaped is not judged here: + * {@link translationLocaleTagCanonicalized} runs first, rewrites the `_` and + * padded spellings into tags and reports the rest. + * + * **A rewrite changes what renders, on purpose.** A `zh-cn` item's strings + * start resolving for `zh-CN` sessions. On a bundle, a key that folds onto a + * key the entry already holds is merged only when the two values are identical + * (`renameKey`); otherwise both are left as stored and reported, because + * picking a winner would silently discard one translator's strings. + * `TranslationBundleSchema` refuses the same pair (`zh-cn` beside `zh-CN`) for + * the same reason. + * + * **Reach.** The surface of {@link translationLocaleTagCanonicalized}: a bare + * item's `locale`, a bundle entry's locale keys, `i18n.defaultLocale`, + * `i18n.fallbackLocale`, `i18n.supportedLocales[]` and `data[].locale[]`. A + * bundle key is rewritten only when it is a tag in another case, so a legacy + * item's group names (`globalActions`, `dashboards`), which are not tags, are + * never touched. + * + * Retired from the load path from day one: the parse folds case for a live + * author itself. Stored rows (`applyConversionsToStoredItem`) and a built + * artifact inside its declared-floor window get the rewrite. Idempotent: a + * canonical tag is never touched again. + */ +const translationLocaleTagCaseFolded: MetadataConversion = { + id: 'translation-locale-tag-case-folded', + toMajor: 18, + retiredFromLoadPath: true, + retiredAfter: '18.2.0', + surface: + 'translation.locale / stack.translations[]. / i18n.defaultLocale / i18n.fallbackLocale / ' + + 'i18n.supportedLocales[] / data[].locale[]', + summary: + "a locale tag written in another case than BCP-47's → the same tag in its canonical case " + + "('zh-cn' → 'zh-CN'): no resolver merged it into the canonical locale, so its strings never " + + 'rendered; a bundle key that would land on a key holding different strings is left as stored ' + + 'and reported', + apply(stack, emit, context) { + /** `v` in its canonical case when `v` is tag-shaped, else `undefined` (left to the entry before). */ + const folded = (v: string): string | undefined => { + const parsed = LocaleSchema.safeParse(v); + return parsed.success ? parsed.data : undefined; + }; + /** A locale value: folded, or returned as is when canonical or not a tag. */ + const value = (raw: string, path: string): string => { + const tag = folded(raw); + if (tag === undefined || tag === raw) return raw; + emit({ from: JSON.stringify(raw), to: JSON.stringify(tag), path }); + return tag; + }; + const list = (raw: unknown, path: string): unknown => { + if (!Array.isArray(raw)) return raw; + let changed = false; + const next = raw.map((v, i) => { + if (typeof v !== 'string') return v; + const mapped = value(v, `${path}[${i}]`); + if (mapped !== v) changed = true; + return mapped; + }); + return changed ? next : raw; + }; + + let next = mapCollection(stack, 'translations', (entry, path) => { + // A bare `translation` ITEM: its own `locale`. + if ('locale' in entry) { + if (typeof entry.locale !== 'string') return entry; + const locale = value(entry.locale, `${path}.locale`); + return locale === entry.locale ? entry : { ...entry, locale }; + } + // A bundle entry: locale code → data. + let bundle = entry; + for (const [key, data] of Object.entries(entry)) { + if (!isDict(data)) continue; + const tag = folded(key); + if (tag === undefined || tag === key) continue; + const renamed = renameKey(bundle, key, tag); + if (renamed === null) { + context?.reportTodo?.({ + path: `${path}.${key}`, + from: JSON.stringify(key), + reason: + `The translation bundle locale key is \`${tag}\` in another case, and the entry already holds ` + + `different strings under \`${tag}\`. Left as stored, it is refused where it is parsed, since ` + + '@objectstack/spec 18. Merge the two by hand and keep one key.', + }); + continue; + } + emit({ from: JSON.stringify(key), to: JSON.stringify(tag), path: `${path}.${key}` }); + bundle = renamed; + } + return bundle; + }); + + const i18n = next.i18n; + if (isDict(i18n)) { + let config = i18n; + for (const key of ['defaultLocale', 'fallbackLocale'] as const) { + const raw = config[key]; + if (typeof raw !== 'string') continue; + const mapped = value(raw, `i18n.${key}`); + if (mapped !== raw) config = { ...config, [key]: mapped }; + } + const supported = list(config.supportedLocales, 'i18n.supportedLocales'); + if (supported !== config.supportedLocales) config = { ...config, supportedLocales: supported }; + if (config !== i18n) next = { ...next, i18n: config }; + } + + return mapCollection(next, 'data', (dataset, path) => { + const scoped = list(dataset.locale, `${path}.locale`); + return scoped === dataset.locale ? dataset : { ...dataset, locale: scoped }; + }); + }, + fixture: { + before: { + translations: [ + // A stored item saved in lower case. + { name: 'pt_br', locale: 'pt-br', messages: { commonSave: 'Salvar' } }, + // Already canonical: left alone. + { name: 'zh_cn', locale: 'zh-CN', messages: { commonSave: '保存' } }, + // Not a tag: not judged here (the entry before reports it). + { name: 'qa_bad_locale', locale: 'not a locale!!', messages: { commonSave: 'x' } }, + // A bundle entry: two keys in another case (one of them letters only), + // one canonical tag. + { + 'ko-kr': { messages: { commonSave: '저장' } }, + IT: { messages: { commonSave: 'Salva' } }, + 'zh-CN': { messages: { commonSave: '保存' } }, + }, + // A legacy item without `locale`: its group keys are not tags. + { name: 'fr', globalActions: { export_all: { label: 'Tout exporter' } } }, + ], + i18n: { defaultLocale: 'en-us', supportedLocales: ['en-us', 'zh-CN', 'zh-hans-cn'], fallbackLocale: 'en' }, + data: [{ object: 'crm_lead', locale: ['zh-cn'], records: [] }], + }, + after: { + translations: [ + { name: 'pt_br', locale: 'pt-BR', messages: { commonSave: 'Salvar' } }, + { name: 'zh_cn', locale: 'zh-CN', messages: { commonSave: '保存' } }, + { name: 'qa_bad_locale', locale: 'not a locale!!', messages: { commonSave: 'x' } }, + { + 'zh-CN': { messages: { commonSave: '保存' } }, + 'ko-KR': { messages: { commonSave: '저장' } }, + it: { messages: { commonSave: 'Salva' } }, + }, + { name: 'fr', globalActions: { export_all: { label: 'Tout exporter' } } }, + ], + i18n: { defaultLocale: 'en-US', supportedLocales: ['en-US', 'zh-CN', 'zh-Hans-CN'], fallbackLocale: 'en' }, + data: [{ object: 'crm_lead', locale: ['zh-CN'], records: [] }], + }, + // One per rewritten value: the item, two bundle keys, the default locale, + // two supported locales and the seed scope. + expectedNotices: 7, + }, +}; + /** * `settings` on every APPLICATION-authored translation face — the platform-only * group leaving `stack.translations` with the type split (protocol 18, #15178, @@ -15764,6 +15942,9 @@ const MAJOR_18_CONVERSIONS: readonly OrderedConversion[] = [ { conversion: timeDefaultUtcSuffixDropped, order: 48 }, { conversion: translationComponentSubmitLabelRemoved, order: 12 }, { conversion: translationLocaleTagCanonicalized, order: 76 }, + // AFTER `translationLocaleTagCanonicalized`: an `_` or padded spelling is made a + // tag there, and only a tag's case is folded here (`zh_cn` → `zh-cn` → `zh-CN`). + { conversion: translationLocaleTagCaseFolded, order: 76.5 }, { conversion: translationPerAppSettingsRemoved, order: 34 }, { conversion: translationWidgetSubCaptionRemoved, order: 55 }, { conversion: tursoConfigEncryptionKeyRemoved, order: 74 }, diff --git a/packages/spec/src/migrations/entries/semantic/18.translation-locale-case-canonicalized.ts b/packages/spec/src/migrations/entries/semantic/18.translation-locale-case-canonicalized.ts new file mode 100644 index 00000000000..ccc337e6fd9 --- /dev/null +++ b/packages/spec/src/migrations/entries/semantic/18.translation-locale-case-canonicalized.ts @@ -0,0 +1,48 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import type { SemanticMigration } from '../../types.js'; + +// The case half of the locale-tag family: the D2 conversion +// `translation-locale-tag-case-folded` folds a stored or built locale to its +// BCP-47 case, the parse folds a live author's, and this entry states what +// that changes for a consumer that compared locales by their exact spelling. +export const entry: SemanticMigration = { + id: 'translation-locale-case-canonicalized', + surface: + 'a locale tag written in another case than BCP-47 canonical case (LocaleSchema): a translation ' + + 'item locale, a translation bundle locale key, i18n.defaultLocale / fallbackLocale / ' + + 'supportedLocales, a seed dataset locale scope, the seed loader locale', + replacement: + 'nothing to rewrite: the parse returns the tag in its BCP-47 case (language lower case, script ' + + 'title case, region upper case, every subtag after a singleton lower case), so `zh-cn` reads ' + + 'back as `zh-CN`, `zh-hans-cn` as `zh-Hans-CN` and `EN` as `en`. A consumer that matched the ' + + 'authored spelling exactly matches the canonical one. Two exact matchers compare a folded ' + + 'i18n locale with a row tagged as written, and neither row is folded: the email-template floor ' + + 'warning (warnEmailTemplateLocaleFloor in stack.zod.ts) compares i18n.supportedLocales with ' + + 'emailTemplates[].locale, and the messaging template ladder compares ' + + 'localeCandidates(i18n.getDefaultLocale()) with sys_notification_template.locale. A row tagged ' + + '`zh-cn` no longer matches a supported or default locale authored as `zh-cn`, which now reads ' + + '`zh-CN`: re-tag the row in its canonical case, or fold case in the consumer. The row-side fold ' + + 'lands with the save-door change that stores the parsed locale. A translation bundle holding two ' + + 'keys that fold to one tag (`zh-cn` and `zh-CN`) is refused naming both: merge them by hand ' + + 'under the canonical key', + reason: + 'Case was accepted as written, on the premise that the resolvers fold it. They answer one locale ' + + 'for a request, exact match first, so a translation saved under `zh-cn` beside the stock `zh-CN` ' + + 'bundle was never merged into it: the save answered 200 and a `zh-CN` session kept rendering the ' + + 'stock strings, with no diagnostic. The parse now corrects the case instead of refusing it, so ' + + 'every reader sees one spelling and no input that parsed before is refused, except a bundle ' + + 'whose two keys would now land on one entry and silently drop one set of strings. The fold is ' + + 'case only: it never replaces a legacy subtag the way Intl.getCanonicalLocales does (`iw` stays ' + + '`iw`).', + acceptanceCriteria: + 'Every locale that parses through LocaleSchema reads back in its BCP-47 case. A stored ' + + '`translation` row under `zh-cn` is read back as `zh-CN` by every stored-row seam, so its ' + + 'strings are merged into the `zh-CN` locale a session asks for. A built artifact inside the ' + + 'conversion window (built by @objectstack/spec 18.2.0 or earlier) is folded the same way. A ' + + 'canonical tag is never rewritten. An email-template or notification-template row tagged in ' + + 'another case is matched by its consumer only once the row is re-tagged or the consumer folds ' + + 'case. A bundle holding `zh-cn` beside `zh-CN` fails the parse naming both keys, and the ' + + 'stored-row conversion leaves such a pair as stored and reports it.', + conversionIds: ['translation-locale-tag-case-folded'], +}; diff --git a/packages/spec/src/migrations/entries/semantic/18.translation-locale-non-tag-refused.ts b/packages/spec/src/migrations/entries/semantic/18.translation-locale-non-tag-refused.ts index a7156fb0984..a301dcf45ed 100644 --- a/packages/spec/src/migrations/entries/semantic/18.translation-locale-non-tag-refused.ts +++ b/packages/spec/src/migrations/entries/semantic/18.translation-locale-non-tag-refused.ts @@ -17,7 +17,8 @@ export const entry: SemanticMigration = { + '1-8 letter-or-digit subtags (`en`, `zh-CN`, `zh-Hans-CN`, `es-419`). Write `zh-CN` for ' + '`zh_CN` and drop surrounding whitespace; the conversion does both for stored rows and built ' + 'artifacts. A language name or any other text has no single tag it can have meant, so rewrite ' - + 'it by hand. Case is not judged: `zh-cn` is accepted, because the resolvers fold case', + + 'it by hand. Case is not refused: the parse corrects it (`zh-cn` → `zh-CN`, see ' + + '`translation-locale-case-canonicalized`)', reason: 'LocaleSchema was a bare string, so a translator or AI author who wrote `zh_CN`, a padded ' + 'value or a language name got a green save, and the strings were stored under a locale no ' diff --git a/packages/spec/src/shared/retired-key-removal-version.pin.test.ts b/packages/spec/src/shared/retired-key-removal-version.pin.test.ts index 34c453d7156..e457e363b35 100644 --- a/packages/spec/src/shared/retired-key-removal-version.pin.test.ts +++ b/packages/spec/src/shared/retired-key-removal-version.pin.test.ts @@ -48,7 +48,9 @@ * its surface. A tombstone row reads the key's parse refusal and its * `[REMOVED]` description, which are the two channels. A refusal row reads the * refusal the surface answers, for a value or union-arm retirement that has no - * key to tombstone. Two guards keep the join honest: + * key to tombstone. A todo row reads the TODO a data-at-rest conversion reports + * for a stored shape it leaves as stored, which `os migrate meta` prints to the + * owner. Two guards keep the join honest: * * - every retired entry whose `retiredAfter` is the newest published stable * release must have a row, so a future retirement cannot ship its @@ -95,7 +97,7 @@ import { InterfacePageConfigSchema, PageComponentSchema, PageSchema } from '../u import { AddRecordConfigSchema, FormFieldSchema, FormSectionSchema, FormViewSchema, ListViewSchema, NavigationModeSchema, ObjectListViewSchema, VIEW_METADATA_MEMBERS, ViewScopeSchema, ViewSharingSchema } from '../ui/view.zod'; import { FlowSchema } from '../automation/flow.zod'; import { JobSchema } from '../system/job.zod'; -import { TranslationItemSchema } from '../system/translation.zod'; +import { TranslationBundleSchema, TranslationItemSchema } from '../system/translation.zod'; import { PROTOCOL_MAJOR } from '../kernel/protocol-version'; import { EmailTemplateDefinitionSchema } from '../system/email-template.zod'; import { @@ -220,6 +222,20 @@ const refusal = (where: string, parse: () => Parse, at?: PropertyKey): Renderer return issues.map((issue) => ({ where, text: issue.message })); }; +/** + * The TODO a retired conversion reports for a stored shape it cannot rewrite + * without a guess, run on `stack`: the prescription the owner of a stored row + * meets in `os migrate meta`, which has no parse refusal to read. + */ +const todo = (where: string, id: string, stack: Record): Renderer => () => { + const conversion = ALL_CONVERSIONS.find((c) => c.id === id); + if (!conversion) throw new Error(`${where}: no conversion \`${id}\``); + const reasons: string[] = []; + conversion.apply(stack, () => {}, { reportTodo: (detail) => reasons.push(detail.reason) }); + expect(reasons.length, `${where} must be reported`).toBeGreaterThan(0); + return reasons.map((text) => ({ where, text })); +}; + const ELEMENTS: readonly (readonly [unknown, string, readonly string[]])[] = [ [ElementNumberPropsSchema, 'element:number', ['object', 'filter']], [ElementRecordPickerPropsSchema, 'element:record_picker', ['object', 'filter', 'sort', 'limit']], @@ -280,6 +296,19 @@ const JOIN: Readonly> = { 'locale', ), ], + // #28993: the case fold corrects a lone tag, so what an author meets is the + // collision two keys of one bundle make once folded, at the parse and at rest. + 'translation-locale-tag-case-folded': [ + refusal( + 'a translation bundle holding `zh-cn` beside `zh-CN`', + () => TranslationBundleSchema.safeParse({ 'zh-cn': { messages: { x: 'a' } }, 'zh-CN': { messages: { x: 'b' } } }), + ), + todo( + 'a stored translation bundle holding `zh-cn` beside `zh-CN`', + 'translation-locale-tag-case-folded', + { translations: [{ 'zh-cn': { messages: { x: 'a' } }, 'zh-CN': { messages: { x: 'b' } } }] }, + ), + ], 'manifest-permissions-string-list-removed': [ refusal('manifest.permissions as a list of strings', () => ManifestPermissionsSchema.safeParse(['system.user.read'])), ], diff --git a/packages/spec/src/system/translation.test.ts b/packages/spec/src/system/translation.test.ts index 9733d3675be..700cd3450fd 100644 --- a/packages/spec/src/system/translation.test.ts +++ b/packages/spec/src/system/translation.test.ts @@ -5,6 +5,9 @@ import { TEXT_SLOT_TEMPLATE_REFUSAL } from '../automation/flow-text-slot-templat import { FlowSchema } from '../automation/flow.zod'; import { ScreenConfigSchema, ScreenFieldConfigSchema } from '../automation/builtin-node-config.zod'; import { ALL_CONVERSIONS } from '../conversions/registry'; +import { applyConversionsToStoredItem } from '../conversions/stored'; +import { SeedSchema } from '../data/seed.zod'; +import { SeedLoaderConfigSchema } from '../data/seed-loader.zod'; import { TranslationDataSchema, TranslationBundleSchema, @@ -47,7 +50,7 @@ describe('LocaleSchema', () => { }); }); - it.each(['en', 'zh-CN', 'zh-Hans-CN', 'es-419', 'yue-HK', 'zh-cn'])('accepts the tag %j (case is not judged)', (locale) => { + it.each(['en', 'zh-CN', 'zh-Hans-CN', 'es-419', 'yue-HK', 'zh-cn'])('accepts the tag %j (case is corrected, never refused)', (locale) => { expect(LocaleSchema.safeParse(locale).success).toBe(true); }); @@ -120,6 +123,125 @@ describe('LocaleSchema', () => { }); }); +// #28993: a translation saved under `zh-cn` answered 200 and never reached a +// `zh-CN` session, because a resolver answers one locale, exact match first, +// and the stock `zh-CN` bundle won. The parse now corrects a tag's case. +describe('LocaleSchema corrects a tag to its BCP-47 case (#28993)', () => { + it.each([ + ['zh-cn', 'zh-CN'], + ['ZH-CN', 'zh-CN'], + ['zh-hans-cn', 'zh-Hans-CN'], + ['ZH-HANS-cn', 'zh-Hans-CN'], + ['EN', 'en'], + ['en-us', 'en-US'], + ['yue-hk', 'yue-HK'], + ['sr-LATN-rs', 'sr-Latn-RS'], + ['es-419', 'es-419'], + ['de-1996', 'de-1996'], + // Every subtag after a singleton is lower case (RFC 5646 §2.1.1). + ['en-US-u-CA-Gregory', 'en-US-u-ca-gregory'], + ['en-x-US', 'en-x-us'], + ])('%j parses to %j', (input, canonical) => { + expect(LocaleSchema.parse(input)).toBe(canonical); + }); + + it.each(['en', 'zh-CN', 'zh-Hans-CN', 'es-419', 'pt-BR', 'en-US-u-ca-gregory'])('a canonical tag %j is returned unchanged', (locale) => { + expect(LocaleSchema.parse(locale)).toBe(locale); + }); + + it('folds case only: no legacy-subtag replacement, and no throw on a tag the shape accepts', () => { + // `Intl.getCanonicalLocales` answers `he`, `fil` and `zh` for these, which + // would move a translation to a locale its author never named. + expect(LocaleSchema.parse('iw')).toBe('iw'); + expect(LocaleSchema.parse('TL')).toBe('tl'); + expect(LocaleSchema.parse('cmn-cn')).toBe('cmn-CN'); + // ...and throws on this one, which the shape accepts. + expect(LocaleSchema.parse('zh-hans-hans')).toBe('zh-Hans-Hans'); + }); + + it('every reader of LocaleSchema sees the canonical spelling', () => { + // The item door: `defineTranslation` and the metadata save door's parse. + expect(TranslationItemSchema.parse({ name: 'zh_cn', locale: 'zh-cn', messages: { x: 'y' } }).locale).toBe('zh-CN'); + expect(defineTranslation({ name: 'zh_cn', locale: 'ZH-cn', messages: { x: 'y' } }).locale).toBe('zh-CN'); + // The stack's i18n configuration. + expect(TranslationConfigSchema.parse({ defaultLocale: 'EN', supportedLocales: ['en', 'zh-cn'], fallbackLocale: 'en-us' })) + .toEqual({ defaultLocale: 'en', supportedLocales: ['en', 'zh-CN'], fallbackLocale: 'en-US' }); + // A seed dataset's locale scope and the seed loader's own locale. + expect(SeedSchema.parse({ object: 'crm_lead', locale: ['zh-cn'], records: [] }).locale).toEqual(['zh-CN']); + expect(SeedLoaderConfigSchema.parse({ locale: 'zh-cn' }).locale).toBe('zh-CN'); + }); + + it('a bundle\'s locale keys are folded, in authored order, values untouched', () => { + const data = { messages: { commonSave: '保存' } }; + const parsed = TranslationBundleSchema.parse({ 'zh-cn': data, en: { messages: { commonSave: 'Save' } } }); + expect(Object.keys(parsed)).toEqual(['zh-CN', 'en']); + expect(parsed['zh-CN']).toEqual(data); + expect(Object.keys(defineTranslationBundle({ 'ja-jp': data }))).toEqual(['ja-JP']); + expect(Object.keys(PlatformTranslationBundleSchema.parse({ 'zh-hans-cn': data }))).toEqual(['zh-Hans-CN']); + }); + + it.each([ + ['TranslationBundleSchema', TranslationBundleSchema], + ['PlatformTranslationBundleSchema', PlatformTranslationBundleSchema], + ] as const)('%s refuses two keys that fold to one tag, naming both, instead of dropping one', (_name, schema) => { + const result = schema.safeParse({ + 'zh-cn': { messages: { commonSave: '保存(lower)' } }, + 'zh-CN': { messages: { commonSave: '保存' } }, + }); + expect(result.success).toBe(false); + expect(result.error!.issues.map((i) => [i.code, i.path.join('.')])).toEqual([['custom', 'zh-CN']]); + const message = result.error!.issues[0]!.message; + expect(message).toContain('`zh-cn`'); + expect(message).toContain('`zh-CN`'); + }); + + it('CONTROL: distinct tags in one bundle, and one tag across two bundles, are both accepted', () => { + // Distinct after the fold: nothing collides. + const bundle = { 'zh-CN': { messages: { x: 'a' } }, 'zh-TW': { messages: { x: 'b' } }, 'zh-hans-cn': { messages: { x: 'c' } } }; + expect(Object.keys(TranslationBundleSchema.parse(bundle))).toEqual(['zh-CN', 'zh-TW', 'zh-Hans-CN']); + // Two bundles of `stack.translations` are separate records; the runtime + // deep-merges them per locale, so `zh-cn` in one and `zh-CN` in the other + // is not a collision, it is one locale. + expect(TranslationBundleSchema.safeParse({ 'zh-cn': { messages: { x: 'a' } } }).success).toBe(true); + expect(TranslationBundleSchema.safeParse({ 'zh-CN': { messages: { x: 'b' } } }).success).toBe(true); + }); + + it('data at rest: a stored row under `zh-cn` is read back as `zh-CN` by the stored-row seam', () => { + // `applyConversionsToStoredItem` is what every rehydration seam replays, + // the runtime-authored translation sync included, before it keys the row + // by its locale. The save door stores the authored body, so this is the + // seam that carries the fold to a stored row. + const row = { name: 'zh_cn', locale: 'zh-cn', objects: { project: { label: '项目(lower)' } } }; + expect(applyConversionsToStoredItem('translation', row)).toEqual({ ...row, locale: 'zh-CN' }); + // Canonical rows come back untouched: the pass is idempotent. + const canonical = { name: 'zh_cn', locale: 'zh-CN', messages: { x: 'y' } }; + expect(applyConversionsToStoredItem('translation', canonical)).toEqual(canonical); + // An underscore spelling is made a tag by the entry before and folded by + // the case entry after it: the order is what takes `zh_cn` to `zh-CN`. + const underscored = { name: 'zh_cn', locale: 'zh_cn', messages: { x: 'y' } }; + expect(applyConversionsToStoredItem('translation', underscored)).toEqual({ ...underscored, locale: 'zh-CN' }); + }); + + it('data at rest: the case entry reports a case collision instead of guessing, and merges an identical twin', () => { + const conversion = ALL_CONVERSIONS.find((c) => c.id === 'translation-locale-tag-case-folded')!; + expect(conversion.retiredAfter).toBe('18.2.0'); + const todos: { path: string }[] = []; + const notices: { path: string }[] = []; + const stack = { + translations: [ + // `zh-cn` is `zh-CN` in another case, which the entry already holds with other strings. + { 'zh-cn': { messages: { x: 'a' } }, 'zh-CN': { messages: { x: 'b' } } }, + // An identical twin is merged: nothing is lost. + { 'ja-jp': { messages: { x: 'c' } }, 'ja-JP': { messages: { x: 'c' } } }, + ], + }; + const out = conversion.apply(stack, (n) => notices.push(n), { reportTodo: (t) => todos.push(t) }); + expect(todos.map((t) => t.path)).toEqual(['translations[0].zh-cn']); + expect(notices.map((n) => n.path)).toEqual(['translations[1].ja-jp']); + expect(out.translations).toEqual([stack.translations[0], { 'ja-JP': { messages: { x: 'c' } } }]); + }); +}); + describe('TranslationDataSchema', () => { it('should accept empty translation data', () => { const data = TranslationDataSchema.parse({}); diff --git a/packages/spec/src/system/translation.zod.ts b/packages/spec/src/system/translation.zod.ts index 858a780f280..1d5abd75e4a 100644 --- a/packages/spec/src/system/translation.zod.ts +++ b/packages/spec/src/system/translation.zod.ts @@ -24,8 +24,8 @@ import { textSlotTemplateRefusal } from '../automation/flow-text-slot-template'; * * Shape only, by decision: membership has no registry to check against * (`Intl.getCanonicalLocales('xx-YY')` succeeds, which is why - * `shared/value-domain.zod.ts` dropped a `bcp47_locale` domain), and case is - * not judged because the resolvers fold it (`zh-cn` resolves to `zh-CN`). + * `shared/value-domain.zod.ts` dropped a `bcp47_locale` domain). Case is not + * refused either; it is corrected, see {@link canonicalLocaleCase}. * The primary subtag is held to the 2–3 letters of the ISO 639 codes every * tag in use starts with: BCP-47 reserves 4 letters and registers no 5–8 * letter language, so the only values that wider grammar ever admitted were @@ -42,12 +42,109 @@ const LOCALE_TAG_REFUSAL = + 'is under a locale no session ever asks for, so its strings never render. Separate subtags with ' + '`-`, not `_`, and write the tag, not the name (`zh-CN`, not `Chinese`).'; +const LOCALE_TAG_DESCRIPTION = 'BCP-47 Language Tag (e.g. en-US, zh-CN)'; + +/** + * A tag's BCP-47 case (RFC 5646 §2.1.1): the language subtag lower case, a + * 2-character subtag upper case (the region: `CN`), a 4-character one title + * case (the script: `Hans`), everything else lower case, and every subtag + * after a singleton (`-u-`, `-x-`) lower case. `zh-cn` → `zh-CN`, + * `ZH-HANS-cn` → `zh-Hans-CN`, `EN` → `en`; a canonical tag is returned as is. + * + * Why the parse corrects case instead of leaving it to the resolvers: a + * resolver answers ONE locale for a request, exact match first, so a + * translation stored under `zh-cn` beside the stock `zh-CN` bundle was never + * merged into it; a `zh-CN` session kept rendering the stock strings, with no + * diagnostic (#28993). Folding at the parse gives every reader one spelling, + * and a lowercase author is corrected rather than refused, so nothing that + * parsed before is refused now. + * + * Case only, deliberately not `Intl.getCanonicalLocales`, measured on Node 22: + * that also replaces legacy and macrolanguage subtags (`iw` → `he`, + * `tl` → `fil`, `cmn` → `zh`), which would move a translation to a locale its + * author never named, and it THROWS on tags {@link LOCALE_TAG_SHAPE} accepts + * (`zh-Hans-Hans`, `en-GB-oed`), which would turn a shape refusal into a + * crash. Its output also depends on the runtime's ICU data, where this fold + * is the same string in every runtime the spec ships to. + * + * ⛔ Package-internal: the public answer is a {@link LocaleSchema} parse, and + * an exported fold would be an invitation to fold a second time in a consumer. + */ +function canonicalLocaleCase(tag: string): string { + let afterSingleton = false; + return tag + .split('-') + .map((subtag, index) => { + if (index === 0 || afterSingleton) return subtag.toLowerCase(); + if (subtag.length === 1) { + afterSingleton = true; + return subtag.toLowerCase(); + } + if (subtag.length === 2) return subtag.toUpperCase(); + if (subtag.length === 4) return subtag.charAt(0).toUpperCase() + subtag.slice(1).toLowerCase(); + return subtag.toLowerCase(); + }) + .join('-'); +} + +/** + * The tag shape alone, without the case fold: the KEY of the two bundle + * records below. A record cannot fold its keys one at a time, because two + * keys that fold to one tag would land on one entry and the later one would + * silently replace the earlier; the records judge their keys together + * ({@link refuseLocaleKeysFoldingTogether}) and then fold them + * ({@link foldLocaleKeys}). Same pattern, same refusal and description as + * {@link LocaleSchema}, so the published shape of a bundle key does not move. + */ +const LocaleTagKeySchema = lazySchema(() => z + .string() + .regex(LOCALE_TAG_SHAPE, LOCALE_TAG_REFUSAL) + .describe(LOCALE_TAG_DESCRIPTION)); + export const LocaleSchema = lazySchema(() => z .string() .regex(LOCALE_TAG_SHAPE, LOCALE_TAG_REFUSAL) - .describe('BCP-47 Language Tag (e.g. en-US, zh-CN)')); + .overwrite(canonicalLocaleCase) + .describe(LOCALE_TAG_DESCRIPTION)); export type Locale = z.input; +/** + * Refuses a bundle holding two locale keys that fold to one tag (`zh-cn` and + * `zh-CN`), naming both. Picking a winner would discard one translator's + * strings without a word, which is the policy the stored-row conversion + * (`translation-locale-tag-case-folded`) already follows for the same + * collision. Runs before {@link foldLocaleKeys}, so it sees the keys as written. + */ +function refuseLocaleKeysFoldingTogether(bundle: Record, ctx: z.RefinementCtx): void { + const firstByTag = new Map(); + for (const key of Object.keys(bundle)) { + const tag = canonicalLocaleCase(key); + const first = firstByTag.get(tag); + if (first === undefined) { + firstByTag.set(tag, key); + continue; + } + ctx.addIssue({ + code: 'custom', + path: [key], + message: + `locale keys \`${first}\` and \`${key}\` are one locale, \`${tag}\`. Refused since ` + + "@objectstack/spec 18, because a locale tag's case is corrected on parse, so both entries would " + + 'land on one key and one set of strings would be dropped. Merge the two by hand and keep one ' + + `key, \`${tag}\`.`, + }); + } +} + +/** Every locale key in its BCP-47 case; the input itself when none moves. */ +function foldLocaleKeys(bundle: Record): Record { + const keys = Object.keys(bundle); + if (keys.every((key) => canonicalLocaleCase(key) === key)) return bundle; + const folded: Record = {}; + for (const key of keys) folded[canonicalLocaleCase(key)] = bundle[key] as T; + return folded; +} + /** * Shared history sentence for every shape in this file. * @@ -2364,7 +2461,11 @@ export type ServedTranslationData = z.input; // Translation Bundle (all locales) // ──────────────────────────────────────────────────────────────────────────── -export const TranslationBundleSchema = lazySchema(() => z.record(LocaleSchema, TranslationDataSchema).describe('Map of locale codes to per-app translation data')); +export const TranslationBundleSchema = lazySchema(() => z + .record(LocaleTagKeySchema, TranslationDataSchema) + .superRefine(refuseLocaleKeysFoldingTogether) + .overwrite(foldLocaleKeys) + .describe('Map of locale codes to per-app translation data')); export type TranslationBundle = z.input; @@ -2374,7 +2475,11 @@ export type TranslationBundle = z.input; * the bundle that needs it; a platform package authoring only groups the * per-app face also declares can keep the narrower {@link TranslationBundle}. */ -export const PlatformTranslationBundleSchema = lazySchema(() => z.record(LocaleSchema, PlatformTranslationDataSchema).describe('Map of locale codes to platform translation data')); +export const PlatformTranslationBundleSchema = lazySchema(() => z + .record(LocaleTagKeySchema, PlatformTranslationDataSchema) + .superRefine(refuseLocaleKeysFoldingTogether) + .overwrite(foldLocaleKeys) + .describe('Map of locale codes to platform translation data')); export type PlatformTranslationBundle = z.input;