diff --git a/lib/opencdd/languages.rb b/lib/opencdd/languages.rb index 6d85dc1..1c27533 100644 --- a/lib/opencdd/languages.rb +++ b/lib/opencdd/languages.rb @@ -13,11 +13,21 @@ module Opencdd # to render a language switcher and by the data pipeline to report # language coverage. class Languages + # IEC CDD source .xls files sometimes use non-standard language + # codes that diverge from ISO 639-1. Map them here so the entire + # ecosystem (gem, JSON wire format, browser CSS, TS model) speaks + # the same ISO code. When a non-conformant code is encountered, + # +normalize+ emits a one-line warning on stderr so data-quality + # issues are visible without silently rewriting. + LANG_ALIASES = { + "jp" => "ja", + }.freeze + attr_reader :source, :translations def initialize(source: "en", translations: []) - @source = source.to_s - @translations = Array(translations).map(&:to_s).uniq - [@source] + @source = self.class.normalize(source) + @translations = Array(translations).map { |t| self.class.normalize(t) }.uniq - [@source] freeze end @@ -26,7 +36,7 @@ def all end def include?(lang) - all.include?(lang.to_s) + all.include?(self.class.normalize(lang)) end def empty? @@ -51,6 +61,23 @@ def hash [source, translations].hash end + # Normalize a language code to ISO 639-1. + # + # Returns the input unchanged if it is already standard. If the + # code is a known non-conformant alias (e.g. "jp" from IEC CDD + # source .xls), returns the ISO equivalent ("ja") and emits a + # warning on stderr so the data-quality issue is visible. + def self.normalize(lang) + return lang if lang.nil? + code = lang.to_s.strip + return code if code.empty? + if LANG_ALIASES.key?(code) + warn "[opencdd] non-conformant language code #{code.inspect} → #{LANG_ALIASES[code].inspect} (ISO 639-1)" + return LANG_ALIASES[code] + end + code + end + # Scan a properties hash for +.+ keys and # return a Languages object covering every language seen. The # source language defaults to +default_source+ when no explicit @@ -60,7 +87,7 @@ def self.from_properties(properties, default_source: "en") next unless key.include?(".") prefix, lang = key.split(".", 2) next unless lang =~ /\A[a-z]{2}(-[a-z0-9]+)?\z/i - acc << lang + acc << normalize(lang) end source = langs.include?(default_source) ? default_source : (langs.first || default_source) translations = langs.to_a - [source] diff --git a/lib/opencdd/parcel/sheet_schema.rb b/lib/opencdd/parcel/sheet_schema.rb index 25d6a85..0b66c66 100644 --- a/lib/opencdd/parcel/sheet_schema.rb +++ b/lib/opencdd/parcel/sheet_schema.rb @@ -22,6 +22,8 @@ class SheetSchema # Canonicalize a Parcel column ID. Splits language tags # (.) so they round-trip cleanly. Returns the # canonical ID (or the input unchanged if no mapping exists). + # Language codes are normalized to ISO 639-1 via + # +Opencdd::Languages.normalize+. def self.canonical_id(raw_id) return nil if raw_id.nil? s = raw_id.to_s.strip @@ -30,7 +32,7 @@ def self.canonical_id(raw_id) base = match ? match.pre_match : s lang = match && match[:lang] canonical = VARIANT_TO_CANONICAL[base] || base - lang ? "#{canonical}.#{lang}" : canonical + lang ? "#{canonical}.#{Opencdd::Languages.normalize(lang)}" : canonical end DIRECTIVE_ROWS = %w[ @@ -260,6 +262,7 @@ def lang_hash_for(directive, col_idx) next if v.nil? s = v.to_s.strip next if s.empty? + lang = Opencdd::Languages.normalize(lang) if lang && !lang.empty? lang = "en" if lang.nil? || lang.empty? h[lang] = s end diff --git a/spec/languages_spec.rb b/spec/languages_spec.rb index 6f4ba74..9f5b32b 100644 --- a/spec/languages_spec.rb +++ b/spec/languages_spec.rb @@ -83,6 +83,48 @@ end end + describe ".normalize" do + it "passes through ISO 639-1 codes unchanged" do + expect(described_class.normalize("en")).to eq("en") + expect(described_class.normalize("ja")).to eq("ja") + expect(described_class.normalize("de")).to eq("de") + end + + it "maps jp to ja (IEC CDD non-conformant)" do + expect(described_class.normalize("jp")).to eq("ja") + end + + it "warns on stderr when a non-conformant code is seen" do + expect { described_class.normalize("jp") } + .to output(/non-conformant language code "jp"/).to_stderr + end + + it "does not warn for standard codes" do + expect { described_class.normalize("en") }.not_to output.to_stderr + end + + it "handles nil and empty gracefully" do + expect(described_class.normalize(nil)).to be_nil + expect(described_class.normalize("")).to eq("") + end + + it "strips whitespace" do + expect(described_class.normalize(" en ")).to eq("en") + end + end + + describe "normalization on construction" do + it "normalizes source and translations" do + langs = described_class.new(source: "en", translations: %w[jp fr]) + expect(langs.translations).to eq(%w[ja fr]) + end + + it "does not double-count jp and ja" do + langs = described_class.new(source: "en", translations: %w[jp ja]) + expect(langs.translations).to eq(%w[ja]) + end + end + describe ".from_properties" do it "scans properties hash for language-tagged keys" do props = { "MDC_P004.en" => "Vehicle", "MDC_P004.fr" => "Véhicule", "MDC_P004.de" => "Fahrzeug" } @@ -102,5 +144,11 @@ langs = described_class.from_properties(props) expect(langs.all).to eq(%w[en]) end + + it "normalizes non-conformant codes like jp to ja" do + props = { "MDC_P004.en" => "Vehicle", "MDC_P004.jp" => "車両" } + langs = described_class.from_properties(props) + expect(langs.translations).to eq(%w[ja]) + end end end diff --git a/spec/parcel/sheet_schema_spec.rb b/spec/parcel/sheet_schema_spec.rb index cbe1c4d..cd5b670 100644 --- a/spec/parcel/sheet_schema_spec.rb +++ b/spec/parcel/sheet_schema_spec.rb @@ -49,5 +49,30 @@ s = described_class.from_header_rows(rows) expect(s.find_by_property_id("MDC_P004.fr").name("fr")).to eq("Nom préféré") end + + it "normalizes non-conformant language codes (jp → ja) in column IDs" do + rows = [ + ["#PROPERTY_ID", "MDC_P004_1.en", "MDC_P004_1.jp"], + ["#PROPERTY_NAME.en", "Preferred name", nil], + ["#PROPERTY_NAME.jp", nil, "推奨名"], + ["#DATATYPE", "STRING_TYPE", "TRANSLATABLE_STRING_TYPE"], + ["#REQUIREMENT", "MAND", "MAND"], + ] + s = described_class.from_header_rows(rows) + expect(s.columns.map(&:property_id)) + .to eq(["MDC_P004.en", "MDC_P004.ja"]) + col = s.find_by_property_id("MDC_P004.ja") + expect(col.name("ja")).to eq("推奨名") + end + end + + describe ".canonical_id" do + it "normalizes jp suffix to ja" do + expect(described_class.canonical_id("MDC_P004.jp")).to eq("MDC_P004.ja") + end + + it "preserves standard language suffixes" do + expect(described_class.canonical_id("MDC_P004.de")).to eq("MDC_P004.de") + end end end