Normalize non-standard language codes to ISO 639-1 at XLS ingestion

lib/opencdd/languages.rb

@@ -12,6 +12,12 @@ module Opencdd
   # "what languages does this dictionary have?" — used by the browser
   # to render a language switcher and by the data pipeline to report
   # language coverage.
+  #
+  # This is a model object. It does not normalize language codes:
+  # source-format quirks (e.g. IEC CDD .xls files using "jp" for
+  # Japanese) are normalized at the SheetSchema ingestion boundary
+  # via +Opencdd::Parcel::LanguageAliases+, so every consumer of
+  # this class can assume ISO 639-1 codes unconditionally.
   class Languages
     attr_reader :source, :translations
 
@@ -58,7 +64,7 @@ def hash
     def self.from_properties(properties, default_source: "en")
       langs = properties.keys.each_with_object(Set.new) do |key, acc|
         next unless key.include?(".")
-        prefix, lang = key.split(".", 2)
+        _, lang = key.split(".", 2)
         next unless lang =~ /\A[a-z]{2}(-[a-z0-9]+)?\z/i
         acc << lang
       end
@@ -67,4 +73,4 @@ def self.from_properties(properties, default_source: "en")
       new(source: source, translations: translations)
     end
   end
-end
+end

lib/opencdd/parcel.rb

@@ -7,6 +7,7 @@ module Parcel
 
     autoload :Metadata,        "opencdd/parcel/metadata"
     autoload :SheetSchema,     "opencdd/parcel/sheet_schema"
+    autoload :LanguageAliases, "opencdd/parcel/language_aliases"
     autoload :Sheet,           "opencdd/parcel/sheet"
     autoload :Workbook,        "opencdd/parcel/workbook"
     autoload :WorkbookReader,  "opencdd/parcel/workbook_reader"

lib/opencdd/parcel/language_aliases.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Parcel
+    # Maps non-conformant language codes found in IEC CDD source .xls
+    # exports to their ISO 639-1 equivalents.
+    #
+    # This is a Parcel-layer concern, not a model concern: the aliases
+    # exist because one specific source format (IEC CDD XLS) uses
+    # non-standard codes in its property ID suffixes and directive
+    # rows. The +Languages+ value object — and every downstream
+    # consumer (JSON wire format, browser, TS model) — speaks ISO
+    # 639-1 unconditionally.
+    #
+    # The normalization happens at the SheetSchema ingestion
+    # boundary so downstream code never sees an alias. SheetSchema
+    # also exposes the set of original-to-canonical mappings it
+    # applied via +#normalized_language_codes+ for data-quality
+    # reporting.
+    module LanguageAliases
+      ALIASES = {
+        "jp" => "ja",
+      }.freeze
+
+      # Returns the ISO 639-1 form of +code+, or the input unchanged
+      # if it is already standard. Pure function: no I/O, no side
+      # effects.
+      def self.normalize(code)
+        return code if code.nil?
+        s = code.to_s.strip
+        return s if s.empty?
+        ALIASES.fetch(s, s)
+      end
+
+      # True when +code+ would be rewritten by +normalize+.
+      def self.alias?(code)
+        return false if code.nil?
+        ALIASES.key?(code.to_s.strip)
+      end
+    end
+  end
+end

lib/opencdd/parcel/sheet_schema.rb

@@ -22,6 +22,8 @@ class SheetSchema
       # Canonicalize a Parcel column ID. Splits language tags
       # (<id>.<lang>) so they round-trip cleanly. Returns the
       # canonical ID (or the input unchanged if no mapping exists).
+      # Language codes are normalized to ISO 639-1 via
+      # +Opencdd::Parcel::LanguageAliases.normalize+.
       def self.canonical_id(raw_id)
         return nil if raw_id.nil?
         s = raw_id.to_s.strip
@@ -30,7 +32,7 @@ def self.canonical_id(raw_id)
         base = match ? match.pre_match : s
         lang = match && match[:lang]
         canonical = VARIANT_TO_CANONICAL[base] || base
-        lang ? "#{canonical}.#{lang}" : canonical
+        lang ? "#{canonical}.#{Opencdd::Parcel::LanguageAliases.normalize(lang)}" : canonical
       end
 
       DIRECTIVE_ROWS = %w[
@@ -110,7 +112,7 @@ def obsolete?
 
       DIRECTIVE_ROW_PREFIX = "#".freeze
 
-      attr_reader :columns, :columns_by_id, :column_directives
+      attr_reader :columns, :columns_by_id, :column_directives, :normalized_language_codes
 
       def initialize
         @columns = []
@@ -194,6 +196,7 @@ def finalize!
           @columns_by_id[col.property_id] = col
         end
 
+        @normalized_language_codes = compute_normalized_language_codes.freeze
         freeze
       end
 
@@ -260,11 +263,34 @@ def lang_hash_for(directive, col_idx)
           next if v.nil?
           s = v.to_s.strip
           next if s.empty?
+          lang = Opencdd::Parcel::LanguageAliases.normalize(lang)
           lang = "en" if lang.nil? || lang.empty?
           h[lang] = s
         end
         h
       end
+
+      # Audit trail of language-code normalizations applied at this
+      # schema's ingestion boundary. Returns a frozen Hash mapping each
+      # original (non-conformant) language code seen in the source to
+      # its ISO 639-1 equivalent. Empty when the source spoke ISO.
+      def compute_normalized_language_codes
+        seen = {}
+        @column_directives.each_key do |key|
+          _, lang = key.to_s.split(".", 2)
+          next if lang.nil? || lang.empty?
+          normalized = Opencdd::Parcel::LanguageAliases.normalize(lang)
+          seen[lang] = normalized if normalized != lang
+        end
+        @columns.each do |col|
+          next if col.raw_property_id.nil?
+          _, lang = col.raw_property_id.split(".", 2)
+          next if lang.nil? || lang.empty?
+          normalized = Opencdd::Parcel::LanguageAliases.normalize(lang)
+          seen[lang] = normalized if normalized != lang
+        end
+        seen
+      end
     end
   end
 end

spec/languages_spec.rb

@@ -103,4 +103,27 @@
       expect(langs.all).to eq(%w[en])
     end
   end
+
+  describe "model contract" do
+    it "does not normalize source — callers must supply ISO 639-1 codes" do
+      # Source-format normalization lives at the ingestion boundary
+      # (Opencdd::Parcel::SheetSchema via LanguageAliases). By the
+      # time a Languages value object is built, codes are already
+      # canonical. Passing "jp" through should NOT be silently
+      # rewritten — that would hide upstream data-quality issues.
+      langs = described_class.new(source: "jp", translations: [])
+      expect(langs.source).to eq("jp")
+    end
+
+    it "does not normalize translations" do
+      langs = described_class.new(source: "en", translations: %w[jp])
+      expect(langs.translations).to eq(%w[jp])
+    end
+
+    it "include? does not rewrite the lookup argument" do
+      langs = described_class.new(source: "en", translations: %w[ja])
+      expect(langs.include?("jp")).to be(false)
+      expect(langs.include?("ja")).to be(true)
+    end
+  end
 end

spec/parcel/language_aliases_spec.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+RSpec.describe Opencdd::Parcel::LanguageAliases do
+  describe ".normalize" do
+    it "passes through ISO 639-1 codes unchanged" do
+      expect(described_class.normalize("en")).to eq("en")
+      expect(described_class.normalize("ja")).to eq("ja")
+      expect(described_class.normalize("de")).to eq("de")
+    end
+
+    it "maps jp to ja (IEC CDD non-conformant)" do
+      expect(described_class.normalize("jp")).to eq("ja")
+    end
+
+    it "handles nil and empty gracefully" do
+      expect(described_class.normalize(nil)).to be_nil
+      expect(described_class.normalize("")).to eq("")
+    end
+
+    it "strips surrounding whitespace" do
+      expect(described_class.normalize("  en  ")).to eq("en")
+      expect(described_class.normalize(" jp ")).to eq("ja")
+    end
+
+    it "is a pure function — does not warn or emit any side effect" do
+      expect { described_class.normalize("jp") }.not_to output.to_stderr
+    end
+
+    it "is idempotent — normalizing an already-normalized code is a no-op" do
+      first = described_class.normalize("jp")
+      second = described_class.normalize(first)
+      expect(second).to eq(first)
+    end
+  end
+
+  describe ".alias?" do
+    it "is true for codes in the alias table" do
+      expect(described_class.alias?("jp")).to be(true)
+    end
+
+    it "is false for ISO 639-1 codes" do
+      expect(described_class.alias?("en")).to be(false)
+      expect(described_class.alias?("ja")).to be(false)
+    end
+
+    it "is false for nil and empty" do
+      expect(described_class.alias?(nil)).to be(false)
+      expect(described_class.alias?("")).to be(false)
+    end
+  end
+
+  describe "ALIASES" do
+    it "is frozen" do
+      expect(described_class::ALIASES).to be_frozen
+    end
+
+    it "maps every key to an ISO 639-1 value" do
+      described_class::ALIASES.each do |original, normalized|
+        expect(original).to match(/\A[a-z]{2}\z/)
+        expect(normalized).to match(/\A[a-z]{2}\z/)
+        expect(original).not_to eq(normalized)
+      end
+    end
+  end
+end

spec/parcel/sheet_schema_spec.rb

@@ -49,5 +49,75 @@
       s = described_class.from_header_rows(rows)
       expect(s.find_by_property_id("MDC_P004.fr").name("fr")).to eq("Nom préféré")
     end
+
+    it "normalizes non-conformant language codes (jp → ja) in column IDs" do
+      rows = [
+        ["#PROPERTY_ID", "MDC_P004_1.en", "MDC_P004_1.jp"],
+        ["#PROPERTY_NAME.en", "Preferred name", nil],
+        ["#PROPERTY_NAME.jp", nil, "推奨名"],
+        ["#DATATYPE", "STRING_TYPE", "TRANSLATABLE_STRING_TYPE"],
+        ["#REQUIREMENT", "MAND", "MAND"],
+      ]
+      s = described_class.from_header_rows(rows)
+      expect(s.columns.map(&:property_id))
+        .to eq(["MDC_P004.en", "MDC_P004.ja"])
+      col = s.find_by_property_id("MDC_P004.ja")
+      expect(col.name("ja")).to eq("推奨名")
+    end
+  end
+
+  describe ".canonical_id" do
+    it "normalizes jp suffix to ja" do
+      expect(described_class.canonical_id("MDC_P004.jp")).to eq("MDC_P004.ja")
+    end
+
+    it "preserves standard language suffixes" do
+      expect(described_class.canonical_id("MDC_P004.de")).to eq("MDC_P004.de")
+    end
+  end
+
+  describe "#normalized_language_codes" do
+    subject(:schema) { described_class.from_header_rows(header_rows) }
+
+    it "is empty when the source spoke ISO 639-1" do
+      expect(schema.normalized_language_codes).to eq({})
+    end
+
+    it "records every non-conformant language code seen in directive keys" do
+      rows = [
+        ["#PROPERTY_ID", "MDC_P004_1.en"],
+        ["#PROPERTY_NAME.en", "Preferred name"],
+        ["#PROPERTY_NAME.jp", "推奨名"],
+        ["#DATATYPE", "TRANSLATABLE_STRING_TYPE"],
+        ["#REQUIREMENT", "MAND"],
+      ]
+      s = described_class.from_header_rows(rows)
+      expect(s.normalized_language_codes).to eq("jp" => "ja")
+    end
+
+    it "records non-conformant codes seen in PROPERTY_ID values" do
+      rows = [
+        ["#PROPERTY_ID", "MDC_P004_1.jp"],
+        ["#PROPERTY_NAME", "Preferred name"],
+        ["#DATATYPE", "TRANSLATABLE_STRING_TYPE"],
+        ["#REQUIREMENT", "MAND"],
+      ]
+      s = described_class.from_header_rows(rows)
+      expect(s.normalized_language_codes).to eq("jp" => "ja")
+    end
+
+    it "returns a frozen hash" do
+      expect(schema.normalized_language_codes).to be_frozen
+    end
+
+    it "dedupes — a code seen 1000 times appears once in the audit" do
+      rows = [["#PROPERTY_ID", "MDC_P004_1.jp"]]
+      rows.concat(1000.times.map { |i| ["#PROPERTY_NAME.jp", "name #{i}"] })
+      rows << ["#DATATYPE", "TRANSLATABLE_STRING_TYPE"]
+      rows << ["#REQUIREMENT", "MAND"]
+      s = described_class.from_header_rows(rows)
+      expect(s.normalized_language_codes.size).to eq(1)
+      expect(s.normalized_language_codes).to eq("jp" => "ja")
+    end
   end
 end