diff --git a/scripts/fetch-data.ts b/scripts/fetch-data.ts index 1859605a..54b5263a 100644 --- a/scripts/fetch-data.ts +++ b/scripts/fetch-data.ts @@ -18,6 +18,7 @@ import { contextFromCwd, runPipeline, committedDataPresent } from "../src/lib/bu import { acquireFromLocal, acquireFromRelease, + normalizeLanguageCodes, fixOceanRunnerIrbis, verifyNoJsxWhitespaceBugs, } from "../src/lib/build/stages"; @@ -35,7 +36,7 @@ const ctx = contextFromCwd(); ctx.log = log; const CDD_DATA_RELEASE = ctx.env?.CDD_DATA_RELEASE; -const fixAndVerify = [fixOceanRunnerIrbis(), verifyNoJsxWhitespaceBugs()]; +const fixAndVerify = [normalizeLanguageCodes(), fixOceanRunnerIrbis(), verifyNoJsxWhitespaceBugs()]; try { // Mode 1: data already committed (CI fast path). Skip acquire. diff --git a/src/components/islands/LanguageSwitcher.vue b/src/components/islands/LanguageSwitcher.vue index ed2c3efa..d94abe23 100644 --- a/src/components/islands/LanguageSwitcher.vue +++ b/src/components/islands/LanguageSwitcher.vue @@ -2,7 +2,7 @@ import { ref, onMounted } from "vue"; const STORAGE_KEY = "opencdd-lang"; -const KNOWN_LANGS = ["en", "de", "fr", "zh"] as const; +const KNOWN_LANGS = ["en", "de", "fr", "ja", "zh"] as const; const current = ref("en"); const available = ref(["en"]); diff --git a/src/lib/build/stages.ts b/src/lib/build/stages.ts index f7dad322..fe617860 100644 --- a/src/lib/build/stages.ts +++ b/src/lib/build/stages.ts @@ -7,8 +7,8 @@ * `execSync`, no shell-out. */ -import { cpSync, existsSync, mkdirSync, rmSync } from "node:fs"; -import { resolve } from "node:path"; +import { cpSync, existsSync, mkdirSync, rmSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { resolve, join } from "node:path"; import { execSync } from "node:child_process"; import { type Stage, @@ -161,6 +161,62 @@ export function fixOceanRunnerIrbis(): Stage { }); } +/** + * Fix: normalize non-standard language codes to ISO 639-1 in all + * database.json files under dataTarget. + * + * IEC CDD source data uses "jp" for Japanese; ISO 639-1 is "ja". + * The browser's CSS visibility rules and LanguageSwitcher expect + * the ISO code. This stage rewrites every *_ml field on every + * entity in every dictionary, renaming the "jp" key to "ja" + * (preserving an existing "ja" key if both are present). Idempotent. + */ +const LANG_ALIASES: Record = { + jp: "ja", +}; + +function normalizeEntityLangs(entity: Record): number { + let changed = 0; + for (const key of Object.keys(entity)) { + if (!key.endsWith("_ml")) continue; + const ml = entity[key]; + if (typeof ml !== "object" || ml === null || Array.isArray(ml)) continue; + for (const [from, to] of Object.entries(LANG_ALIASES)) { + const record = ml as Record; + if (!(from in record)) continue; + if (!(to in record)) { + record[to] = record[from]; + } + delete record[from]; + changed++; + } + } + return changed; +} + +export function normalizeLanguageCodes(): Stage { + return stage("normalize-language-codes", (ctx) => { + let totalRenamed = 0; + let dictCount = 0; + for (const entry of readdirSync(ctx.dataTarget, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const dbPath = join(ctx.dataTarget, entry.name, "database.json"); + if (!existsSync(dbPath)) continue; + const raw = readFileSync(dbPath, "utf8"); + const entities = JSON.parse(raw) as Record[]; + for (const entity of entities) { + totalRenamed += normalizeEntityLangs(entity); + } + writeFileSync(dbPath, JSON.stringify(entities)); + dictCount++; + } + if (totalRenamed === 0) { + return { ok: true, skipped: true, message: "no non-standard codes found" }; + } + return { ok: true, message: `renamed ${totalRenamed} key(s) across ${dictCount} dict(s)` }; + }); +} + /** * Verify: scan all .astro / .mdx files for JSX whitespace bugs (text * immediately followed by an inline opening tag on the next line). @@ -194,6 +250,7 @@ export function localAcquireFixVerify(): Stage[] { return [ skipIfCommitted, acquireFromLocal(), + normalizeLanguageCodes(), fixOceanRunnerIrbis(), verifyNoJsxWhitespaceBugs(), ]; @@ -204,12 +261,10 @@ export function localAcquireFixVerify(): Stage[] { * fetches, with local-copy fallback on 404. */ export function releaseAcquireFixVerify(): Stage[] { - // The release stage includes its own 404 fallback inside the runner; - // see acquireFromRelease. Tests can compose stages differently if - // they want to assert the fallback path explicitly. return [ skipIfCommitted, acquireFromRelease(), + normalizeLanguageCodes(), fixOceanRunnerIrbis(), verifyNoJsxWhitespaceBugs(), ]; diff --git a/tests/lib/pipeline.test.ts b/tests/lib/pipeline.test.ts index 0db00e9f..10f60189 100644 --- a/tests/lib/pipeline.test.ts +++ b/tests/lib/pipeline.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -9,7 +9,7 @@ import { type StageContext, committedDataPresent, } from "~/lib/build/pipeline"; -import { skipIfCommitted, acquireFromLocal, fixOceanRunnerIrbis } from "~/lib/build/stages"; +import { skipIfCommitted, acquireFromLocal, fixOceanRunnerIrbis, normalizeLanguageCodes } from "~/lib/build/stages"; function makeTempContext(): { ctx: StageContext; cleanup: () => void } { const tmp = mkdtempSync(join(tmpdir(), "opencdd-pipeline-")); @@ -159,4 +159,90 @@ describe("BuildPipeline", () => { }); }); }); + + describe("normalizeLanguageCodes stage", () => { + let ctx: StageContext; + let cleanup!: () => void; + + beforeEach(() => { ({ ctx, cleanup } = makeTempContext()); }); + afterEach(() => cleanup()); + + it("renames jp to ja on all _ml fields in database.json", async () => { + mkdirSync(join(ctx.dataTarget, "mldict"), { recursive: true }); + const nodes = [ + { + irdi: "X#ACE061", + code: "ACE061", + type: "property", + preferred_name: "mean operating time to failure", + preferred_name_ml: { + en: "mean operating time to failure", + de: "mittlere Betriebszeit bis zum Ausfall", + fr: "durée moyenne de fonctionnement avant défaillance", + jp: "平均故障間動作時間", + zh: "平均失效前工作时间", + }, + definition_ml: { + en: "expectation of the operating time to failure", + jp: "故障までの平均動作時間", + }, + short_name_ml: { en: "MTTF", jp: "MTTF" }, + }, + ]; + const dbPath = join(ctx.dataTarget, "mldict", "database.json"); + writeFileSync(dbPath, JSON.stringify(nodes)); + + const result = await normalizeLanguageCodes().run(ctx); + expect(result.ok).toBe(true); + + const after = JSON.parse(readFileSync(dbPath, "utf8")) as Array>; + const entity = after[0]!; + const pnml = entity.preferred_name_ml as Record; + expect(pnml.ja).toBe("平均故障間動作時間"); + expect(pnml.jp).toBeUndefined(); + expect(pnml.en).toBe("mean operating time to failure"); + + const dml = entity.definition_ml as Record; + expect(dml.ja).toBe("故障までの平均動作時間"); + expect(dml.jp).toBeUndefined(); + + const snml = entity.short_name_ml as Record; + expect(snml.ja).toBe("MTTF"); + expect(snml.jp).toBeUndefined(); + }); + + it("preserves an existing ja key when both jp and ja are present", async () => { + mkdirSync(join(ctx.dataTarget, "mldict"), { recursive: true }); + const nodes = [ + { + irdi: "X#C", code: "C", type: "class", + preferred_name: "test", + preferred_name_ml: { ja: "正しい", jp: "間違い" }, + }, + ]; + const dbPath = join(ctx.dataTarget, "mldict", "database.json"); + writeFileSync(dbPath, JSON.stringify(nodes)); + + await normalizeLanguageCodes().run(ctx); + + const after = JSON.parse(readFileSync(dbPath, "utf8")) as Array>; + const pnml = after[0]!.preferred_name_ml as Record; + expect(pnml.ja).toBe("正しい"); + expect(pnml.jp).toBeUndefined(); + }); + + it("skips when no non-standard codes are found", async () => { + mkdirSync(join(ctx.dataTarget, "cleandict"), { recursive: true }); + writeFileSync( + join(ctx.dataTarget, "cleandict", "database.json"), + JSON.stringify([ + { irdi: "X#A", code: "A", type: "class", preferred_name_ml: { en: "hello", ja: "こんにちは" } }, + ]), + ); + + const result = await normalizeLanguageCodes().run(ctx); + expect(result.ok).toBe(true); + expect(result).toHaveProperty("skipped", true); + }); + }); });