Normalize Japanese language code jp to ISO 639-1 ja

scripts/fetch-data.ts

@@ -18,6 +18,7 @@ import { contextFromCwd, runPipeline, committedDataPresent } from "../src/lib/bu
 import {
   acquireFromLocal,
   acquireFromRelease,
+  normalizeLanguageCodes,
   fixOceanRunnerIrbis,
   verifyNoJsxWhitespaceBugs,
 } from "../src/lib/build/stages";
@@ -35,7 +36,7 @@ const ctx = contextFromCwd();
 ctx.log = log;
 const CDD_DATA_RELEASE = ctx.env?.CDD_DATA_RELEASE;
 
-const fixAndVerify = [fixOceanRunnerIrbis(), verifyNoJsxWhitespaceBugs()];
+const fixAndVerify = [normalizeLanguageCodes(), fixOceanRunnerIrbis(), verifyNoJsxWhitespaceBugs()];
 
 try {
   // Mode 1: data already committed (CI fast path). Skip acquire.

src/components/islands/LanguageSwitcher.vue

@@ -2,7 +2,7 @@
 import { ref, onMounted } from "vue";
 
 const STORAGE_KEY = "opencdd-lang";
-const KNOWN_LANGS = ["en", "de", "fr", "zh"] as const;
+const KNOWN_LANGS = ["en", "de", "fr", "ja", "zh"] as const;
 
 const current = ref<string>("en");
 const available = ref<string[]>(["en"]);

src/lib/build/stages.ts

@@ -7,8 +7,8 @@
  * `execSync`, no shell-out.
  */
 
-import { cpSync, existsSync, mkdirSync, rmSync } from "node:fs";
-import { resolve } from "node:path";
+import { cpSync, existsSync, mkdirSync, rmSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { resolve, join } from "node:path";
 import { execSync } from "node:child_process";
 import {
   type Stage,
@@ -161,6 +161,62 @@ export function fixOceanRunnerIrbis(): Stage {
   });
 }
 
+/**
+ * Fix: normalize non-standard language codes to ISO 639-1 in all
+ * database.json files under dataTarget.
+ *
+ * IEC CDD source data uses "jp" for Japanese; ISO 639-1 is "ja".
+ * The browser's CSS visibility rules and LanguageSwitcher expect
+ * the ISO code. This stage rewrites every *_ml field on every
+ * entity in every dictionary, renaming the "jp" key to "ja"
+ * (preserving an existing "ja" key if both are present). Idempotent.
+ */
+const LANG_ALIASES: Record<string, string> = {
+  jp: "ja",
+};
+
+function normalizeEntityLangs(entity: Record<string, unknown>): number {
+  let changed = 0;
+  for (const key of Object.keys(entity)) {
+    if (!key.endsWith("_ml")) continue;
+    const ml = entity[key];
+    if (typeof ml !== "object" || ml === null || Array.isArray(ml)) continue;
+    for (const [from, to] of Object.entries(LANG_ALIASES)) {
+      const record = ml as Record<string, unknown>;
+      if (!(from in record)) continue;
+      if (!(to in record)) {
+        record[to] = record[from];
+      }
+      delete record[from];
+      changed++;
+    }
+  }
+  return changed;
+}
+
+export function normalizeLanguageCodes(): Stage {
+  return stage("normalize-language-codes", (ctx) => {
+    let totalRenamed = 0;
+    let dictCount = 0;
+    for (const entry of readdirSync(ctx.dataTarget, { withFileTypes: true })) {
+      if (!entry.isDirectory()) continue;
+      const dbPath = join(ctx.dataTarget, entry.name, "database.json");
+      if (!existsSync(dbPath)) continue;
+      const raw = readFileSync(dbPath, "utf8");
+      const entities = JSON.parse(raw) as Record<string, unknown>[];
+      for (const entity of entities) {
+        totalRenamed += normalizeEntityLangs(entity);
+      }
+      writeFileSync(dbPath, JSON.stringify(entities));
+      dictCount++;
+    }
+    if (totalRenamed === 0) {
+      return { ok: true, skipped: true, message: "no non-standard codes found" };
+    }
+    return { ok: true, message: `renamed ${totalRenamed} key(s) across ${dictCount} dict(s)` };
+  });
+}
+
 /**
  * Verify: scan all .astro / .mdx files for JSX whitespace bugs (text
  * immediately followed by an inline opening tag on the next line).
@@ -194,6 +250,7 @@ export function localAcquireFixVerify(): Stage[] {
   return [
     skipIfCommitted,
     acquireFromLocal(),
+    normalizeLanguageCodes(),
     fixOceanRunnerIrbis(),
     verifyNoJsxWhitespaceBugs(),
   ];
@@ -204,12 +261,10 @@ export function localAcquireFixVerify(): Stage[] {
  * fetches, with local-copy fallback on 404.
  */
 export function releaseAcquireFixVerify(): Stage[] {
-  // The release stage includes its own 404 fallback inside the runner;
-  // see acquireFromRelease. Tests can compose stages differently if
-  // they want to assert the fallback path explicitly.
   return [
     skipIfCommitted,
     acquireFromRelease(),
+    normalizeLanguageCodes(),
     fixOceanRunnerIrbis(),
     verifyNoJsxWhitespaceBugs(),
   ];

tests/lib/pipeline.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
+import { mkdtempSync, rmSync, writeFileSync, mkdirSync, readFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
@@ -9,7 +9,7 @@ import {
   type StageContext,
   committedDataPresent,
 } from "~/lib/build/pipeline";
-import { skipIfCommitted, acquireFromLocal, fixOceanRunnerIrbis } from "~/lib/build/stages";
+import { skipIfCommitted, acquireFromLocal, fixOceanRunnerIrbis, normalizeLanguageCodes } from "~/lib/build/stages";
 
 function makeTempContext(): { ctx: StageContext; cleanup: () => void } {
   const tmp = mkdtempSync(join(tmpdir(), "opencdd-pipeline-"));
@@ -159,4 +159,90 @@ describe("BuildPipeline", () => {
       });
     });
   });
+
+  describe("normalizeLanguageCodes stage", () => {
+    let ctx: StageContext;
+    let cleanup!: () => void;
+
+    beforeEach(() => { ({ ctx, cleanup } = makeTempContext()); });
+    afterEach(() => cleanup());
+
+    it("renames jp to ja on all _ml fields in database.json", async () => {
+      mkdirSync(join(ctx.dataTarget, "mldict"), { recursive: true });
+      const nodes = [
+        {
+          irdi: "X#ACE061",
+          code: "ACE061",
+          type: "property",
+          preferred_name: "mean operating time to failure",
+          preferred_name_ml: {
+            en: "mean operating time to failure",
+            de: "mittlere Betriebszeit bis zum Ausfall",
+            fr: "durée moyenne de fonctionnement avant défaillance",
+            jp: "平均故障間動作時間",
+            zh: "平均失效前工作时间",
+          },
+          definition_ml: {
+            en: "expectation of the operating time to failure",
+            jp: "故障までの平均動作時間",
+          },
+          short_name_ml: { en: "MTTF", jp: "MTTF" },
+        },
+      ];
+      const dbPath = join(ctx.dataTarget, "mldict", "database.json");
+      writeFileSync(dbPath, JSON.stringify(nodes));
+
+      const result = await normalizeLanguageCodes().run(ctx);
+      expect(result.ok).toBe(true);
+
+      const after = JSON.parse(readFileSync(dbPath, "utf8")) as Array<Record<string, unknown>>;
+      const entity = after[0]!;
+      const pnml = entity.preferred_name_ml as Record<string, string>;
+      expect(pnml.ja).toBe("平均故障間動作時間");
+      expect(pnml.jp).toBeUndefined();
+      expect(pnml.en).toBe("mean operating time to failure");
+
+      const dml = entity.definition_ml as Record<string, string>;
+      expect(dml.ja).toBe("故障までの平均動作時間");
+      expect(dml.jp).toBeUndefined();
+
+      const snml = entity.short_name_ml as Record<string, string>;
+      expect(snml.ja).toBe("MTTF");
+      expect(snml.jp).toBeUndefined();
+    });
+
+    it("preserves an existing ja key when both jp and ja are present", async () => {
+      mkdirSync(join(ctx.dataTarget, "mldict"), { recursive: true });
+      const nodes = [
+        {
+          irdi: "X#C", code: "C", type: "class",
+          preferred_name: "test",
+          preferred_name_ml: { ja: "正しい", jp: "間違い" },
+        },
+      ];
+      const dbPath = join(ctx.dataTarget, "mldict", "database.json");
+      writeFileSync(dbPath, JSON.stringify(nodes));
+
+      await normalizeLanguageCodes().run(ctx);
+
+      const after = JSON.parse(readFileSync(dbPath, "utf8")) as Array<Record<string, unknown>>;
+      const pnml = after[0]!.preferred_name_ml as Record<string, string>;
+      expect(pnml.ja).toBe("正しい");
+      expect(pnml.jp).toBeUndefined();
+    });
+
+    it("skips when no non-standard codes are found", async () => {
+      mkdirSync(join(ctx.dataTarget, "cleandict"), { recursive: true });
+      writeFileSync(
+        join(ctx.dataTarget, "cleandict", "database.json"),
+        JSON.stringify([
+          { irdi: "X#A", code: "A", type: "class", preferred_name_ml: { en: "hello", ja: "こんにちは" } },
+        ]),
+      );
+
+      const result = await normalizeLanguageCodes().run(ctx);
+      expect(result.ok).toBe(true);
+      expect(result).toHaveProperty("skipped", true);
+    });
+  });
 });