Register DET classification entity type for search-export flow

lib/opencdd/languages.rb

@@ -13,11 +13,21 @@ module Opencdd
   # to render a language switcher and by the data pipeline to report
   # language coverage.
   class Languages
+    # IEC CDD source .xls files sometimes use non-standard language
+    # codes that diverge from ISO 639-1. Map them here so the entire
+    # ecosystem (gem, JSON wire format, browser CSS, TS model) speaks
+    # the same ISO code. When a non-conformant code is encountered,
+    # +normalize+ emits a one-line warning on stderr so data-quality
+    # issues are visible without silently rewriting.
+    LANG_ALIASES = {
+      "jp" => "ja",
+    }.freeze
+
     attr_reader :source, :translations
 
     def initialize(source: "en", translations: [])
-      @source = source.to_s
-      @translations = Array(translations).map(&:to_s).uniq - [@source]
+      @source = self.class.normalize(source)
+      @translations = Array(translations).map { |t| self.class.normalize(t) }.uniq - [@source]
       freeze
     end
 
@@ -26,7 +36,7 @@ def all
     end
 
     def include?(lang)
-      all.include?(lang.to_s)
+      all.include?(self.class.normalize(lang))
     end
 
     def empty?
@@ -51,6 +61,23 @@ def hash
       [source, translations].hash
     end
 
+    # Normalize a language code to ISO 639-1.
+    #
+    # Returns the input unchanged if it is already standard. If the
+    # code is a known non-conformant alias (e.g. "jp" from IEC CDD
+    # source .xls), returns the ISO equivalent ("ja") and emits a
+    # warning on stderr so the data-quality issue is visible.
+    def self.normalize(lang)
+      return lang if lang.nil?
+      code = lang.to_s.strip
+      return code if code.empty?
+      if LANG_ALIASES.key?(code)
+        warn "[opencdd] non-conformant language code #{code.inspect} → #{LANG_ALIASES[code].inspect} (ISO 639-1)"
+        return LANG_ALIASES[code]
+      end
+      code
+    end
+
     # Scan a properties hash for +<property_id>.<lang>+ keys and
     # return a Languages object covering every language seen. The
     # source language defaults to +default_source+ when no explicit
@@ -60,7 +87,7 @@ def self.from_properties(properties, default_source: "en")
         next unless key.include?(".")
         prefix, lang = key.split(".", 2)
         next unless lang =~ /\A[a-z]{2}(-[a-z0-9]+)?\z/i
-        acc << lang
+        acc << normalize(lang)
       end
       source = langs.include?(default_source) ? default_source : (langs.first || default_source)
       translations = langs.to_a - [source]

lib/opencdd/parcel/sheet_schema.rb

@@ -22,6 +22,8 @@ class SheetSchema
       # Canonicalize a Parcel column ID. Splits language tags
       # (<id>.<lang>) so they round-trip cleanly. Returns the
       # canonical ID (or the input unchanged if no mapping exists).
+      # Language codes are normalized to ISO 639-1 via
+      # +Opencdd::Languages.normalize+.
       def self.canonical_id(raw_id)
         return nil if raw_id.nil?
         s = raw_id.to_s.strip
@@ -30,7 +32,7 @@ def self.canonical_id(raw_id)
         base = match ? match.pre_match : s
         lang = match && match[:lang]
         canonical = VARIANT_TO_CANONICAL[base] || base
-        lang ? "#{canonical}.#{lang}" : canonical
+        lang ? "#{canonical}.#{Opencdd::Languages.normalize(lang)}" : canonical
       end
 
       DIRECTIVE_ROWS = %w[
@@ -260,6 +262,7 @@ def lang_hash_for(directive, col_idx)
           next if v.nil?
           s = v.to_s.strip
           next if s.empty?
+          lang = Opencdd::Languages.normalize(lang) if lang && !lang.empty?
           lang = "en" if lang.nil? || lang.empty?
           h[lang] = s
         end

spec/languages_spec.rb

@@ -83,6 +83,48 @@
     end
   end
 
+  describe ".normalize" do
+    it "passes through ISO 639-1 codes unchanged" do
+      expect(described_class.normalize("en")).to eq("en")
+      expect(described_class.normalize("ja")).to eq("ja")
+      expect(described_class.normalize("de")).to eq("de")
+    end
+
+    it "maps jp to ja (IEC CDD non-conformant)" do
+      expect(described_class.normalize("jp")).to eq("ja")
+    end
+
+    it "warns on stderr when a non-conformant code is seen" do
+      expect { described_class.normalize("jp") }
+        .to output(/non-conformant language code "jp"/).to_stderr
+    end
+
+    it "does not warn for standard codes" do
+      expect { described_class.normalize("en") }.not_to output.to_stderr
+    end
+
+    it "handles nil and empty gracefully" do
+      expect(described_class.normalize(nil)).to be_nil
+      expect(described_class.normalize("")).to eq("")
+    end
+
+    it "strips whitespace" do
+      expect(described_class.normalize("  en  ")).to eq("en")
+    end
+  end
+
+  describe "normalization on construction" do
+    it "normalizes source and translations" do
+      langs = described_class.new(source: "en", translations: %w[jp fr])
+      expect(langs.translations).to eq(%w[ja fr])
+    end
+
+    it "does not double-count jp and ja" do
+      langs = described_class.new(source: "en", translations: %w[jp ja])
+      expect(langs.translations).to eq(%w[ja])
+    end
+  end
+
   describe ".from_properties" do
     it "scans properties hash for language-tagged keys" do
       props = { "MDC_P004.en" => "Vehicle", "MDC_P004.fr" => "Véhicule", "MDC_P004.de" => "Fahrzeug" }
@@ -102,5 +144,11 @@
       langs = described_class.from_properties(props)
       expect(langs.all).to eq(%w[en])
     end
+
+    it "normalizes non-conformant codes like jp to ja" do
+      props = { "MDC_P004.en" => "Vehicle", "MDC_P004.jp" => "車両" }
+      langs = described_class.from_properties(props)
+      expect(langs.translations).to eq(%w[ja])
+    end
   end
 end

spec/parcel/sheet_schema_spec.rb

@@ -49,5 +49,30 @@
       s = described_class.from_header_rows(rows)
       expect(s.find_by_property_id("MDC_P004.fr").name("fr")).to eq("Nom préféré")
     end
+
+    it "normalizes non-conformant language codes (jp → ja) in column IDs" do
+      rows = [
+        ["#PROPERTY_ID", "MDC_P004_1.en", "MDC_P004_1.jp"],
+        ["#PROPERTY_NAME.en", "Preferred name", nil],
+        ["#PROPERTY_NAME.jp", nil, "推奨名"],
+        ["#DATATYPE", "STRING_TYPE", "TRANSLATABLE_STRING_TYPE"],
+        ["#REQUIREMENT", "MAND", "MAND"],
+      ]
+      s = described_class.from_header_rows(rows)
+      expect(s.columns.map(&:property_id))
+        .to eq(["MDC_P004.en", "MDC_P004.ja"])
+      col = s.find_by_property_id("MDC_P004.ja")
+      expect(col.name("ja")).to eq("推奨名")
+    end
+  end
+
+  describe ".canonical_id" do
+    it "normalizes jp suffix to ja" do
+      expect(described_class.canonical_id("MDC_P004.jp")).to eq("MDC_P004.ja")
+    end
+
+    it "preserves standard language suffixes" do
+      expect(described_class.canonical_id("MDC_P004.de")).to eq("MDC_P004.de")
+    end
   end
 end