diff --git a/lib/opencdd/languages.rb b/lib/opencdd/languages.rb index 6d85dc1..fa8f4ae 100644 --- a/lib/opencdd/languages.rb +++ b/lib/opencdd/languages.rb @@ -12,6 +12,12 @@ module Opencdd # "what languages does this dictionary have?" — used by the browser # to render a language switcher and by the data pipeline to report # language coverage. + # + # This is a model object. It does not normalize language codes: + # source-format quirks (e.g. IEC CDD .xls files using "jp" for + # Japanese) are normalized at the SheetSchema ingestion boundary + # via +Opencdd::Parcel::LanguageAliases+, so every consumer of + # this class can assume ISO 639-1 codes unconditionally. class Languages attr_reader :source, :translations @@ -58,7 +64,7 @@ def hash def self.from_properties(properties, default_source: "en") langs = properties.keys.each_with_object(Set.new) do |key, acc| next unless key.include?(".") - prefix, lang = key.split(".", 2) + _, lang = key.split(".", 2) next unless lang =~ /\A[a-z]{2}(-[a-z0-9]+)?\z/i acc << lang end @@ -67,4 +73,4 @@ def self.from_properties(properties, default_source: "en") new(source: source, translations: translations) end end -end +end \ No newline at end of file diff --git a/lib/opencdd/parcel.rb b/lib/opencdd/parcel.rb index 69e945d..2c6a910 100644 --- a/lib/opencdd/parcel.rb +++ b/lib/opencdd/parcel.rb @@ -7,6 +7,7 @@ module Parcel autoload :Metadata, "opencdd/parcel/metadata" autoload :SheetSchema, "opencdd/parcel/sheet_schema" + autoload :LanguageAliases, "opencdd/parcel/language_aliases" autoload :Sheet, "opencdd/parcel/sheet" autoload :Workbook, "opencdd/parcel/workbook" autoload :WorkbookReader, "opencdd/parcel/workbook_reader" diff --git a/lib/opencdd/parcel/language_aliases.rb b/lib/opencdd/parcel/language_aliases.rb new file mode 100644 index 0000000..ea3c492 --- /dev/null +++ b/lib/opencdd/parcel/language_aliases.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Opencdd + module Parcel + # Maps non-conformant language codes found in IEC CDD source .xls + # exports to their ISO 639-1 equivalents. + # + # This is a Parcel-layer concern, not a model concern: the aliases + # exist because one specific source format (IEC CDD XLS) uses + # non-standard codes in its property ID suffixes and directive + # rows. The +Languages+ value object — and every downstream + # consumer (JSON wire format, browser, TS model) — speaks ISO + # 639-1 unconditionally. + # + # The normalization happens at the SheetSchema ingestion + # boundary so downstream code never sees an alias. SheetSchema + # also exposes the set of original-to-canonical mappings it + # applied via +#normalized_language_codes+ for data-quality + # reporting. + module LanguageAliases + ALIASES = { + "jp" => "ja", + }.freeze + + # Returns the ISO 639-1 form of +code+, or the input unchanged + # if it is already standard. Pure function: no I/O, no side + # effects. + def self.normalize(code) + return code if code.nil? + s = code.to_s.strip + return s if s.empty? + ALIASES.fetch(s, s) + end + + # True when +code+ would be rewritten by +normalize+. + def self.alias?(code) + return false if code.nil? + ALIASES.key?(code.to_s.strip) + end + end + end +end \ No newline at end of file diff --git a/lib/opencdd/parcel/sheet_schema.rb b/lib/opencdd/parcel/sheet_schema.rb index 25d6a85..af346ee 100644 --- a/lib/opencdd/parcel/sheet_schema.rb +++ b/lib/opencdd/parcel/sheet_schema.rb @@ -22,6 +22,8 @@ class SheetSchema # Canonicalize a Parcel column ID. Splits language tags # (.) so they round-trip cleanly. Returns the # canonical ID (or the input unchanged if no mapping exists). + # Language codes are normalized to ISO 639-1 via + # +Opencdd::Parcel::LanguageAliases.normalize+. def self.canonical_id(raw_id) return nil if raw_id.nil? s = raw_id.to_s.strip @@ -30,7 +32,7 @@ def self.canonical_id(raw_id) base = match ? match.pre_match : s lang = match && match[:lang] canonical = VARIANT_TO_CANONICAL[base] || base - lang ? "#{canonical}.#{lang}" : canonical + lang ? "#{canonical}.#{Opencdd::Parcel::LanguageAliases.normalize(lang)}" : canonical end DIRECTIVE_ROWS = %w[ @@ -110,7 +112,7 @@ def obsolete? DIRECTIVE_ROW_PREFIX = "#".freeze - attr_reader :columns, :columns_by_id, :column_directives + attr_reader :columns, :columns_by_id, :column_directives, :normalized_language_codes def initialize @columns = [] @@ -194,6 +196,7 @@ def finalize! @columns_by_id[col.property_id] = col end + @normalized_language_codes = compute_normalized_language_codes.freeze freeze end @@ -260,11 +263,34 @@ def lang_hash_for(directive, col_idx) next if v.nil? s = v.to_s.strip next if s.empty? + lang = Opencdd::Parcel::LanguageAliases.normalize(lang) lang = "en" if lang.nil? || lang.empty? h[lang] = s end h end + + # Audit trail of language-code normalizations applied at this + # schema's ingestion boundary. Returns a frozen Hash mapping each + # original (non-conformant) language code seen in the source to + # its ISO 639-1 equivalent. Empty when the source spoke ISO. + def compute_normalized_language_codes + seen = {} + @column_directives.each_key do |key| + _, lang = key.to_s.split(".", 2) + next if lang.nil? || lang.empty? + normalized = Opencdd::Parcel::LanguageAliases.normalize(lang) + seen[lang] = normalized if normalized != lang + end + @columns.each do |col| + next if col.raw_property_id.nil? + _, lang = col.raw_property_id.split(".", 2) + next if lang.nil? || lang.empty? + normalized = Opencdd::Parcel::LanguageAliases.normalize(lang) + seen[lang] = normalized if normalized != lang + end + seen + end end end end diff --git a/spec/languages_spec.rb b/spec/languages_spec.rb index 6f4ba74..83e33da 100644 --- a/spec/languages_spec.rb +++ b/spec/languages_spec.rb @@ -103,4 +103,27 @@ expect(langs.all).to eq(%w[en]) end end + + describe "model contract" do + it "does not normalize source — callers must supply ISO 639-1 codes" do + # Source-format normalization lives at the ingestion boundary + # (Opencdd::Parcel::SheetSchema via LanguageAliases). By the + # time a Languages value object is built, codes are already + # canonical. Passing "jp" through should NOT be silently + # rewritten — that would hide upstream data-quality issues. + langs = described_class.new(source: "jp", translations: []) + expect(langs.source).to eq("jp") + end + + it "does not normalize translations" do + langs = described_class.new(source: "en", translations: %w[jp]) + expect(langs.translations).to eq(%w[jp]) + end + + it "include? does not rewrite the lookup argument" do + langs = described_class.new(source: "en", translations: %w[ja]) + expect(langs.include?("jp")).to be(false) + expect(langs.include?("ja")).to be(true) + end + end end diff --git a/spec/parcel/language_aliases_spec.rb b/spec/parcel/language_aliases_spec.rb new file mode 100644 index 0000000..e443a6c --- /dev/null +++ b/spec/parcel/language_aliases_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Opencdd::Parcel::LanguageAliases do + describe ".normalize" do + it "passes through ISO 639-1 codes unchanged" do + expect(described_class.normalize("en")).to eq("en") + expect(described_class.normalize("ja")).to eq("ja") + expect(described_class.normalize("de")).to eq("de") + end + + it "maps jp to ja (IEC CDD non-conformant)" do + expect(described_class.normalize("jp")).to eq("ja") + end + + it "handles nil and empty gracefully" do + expect(described_class.normalize(nil)).to be_nil + expect(described_class.normalize("")).to eq("") + end + + it "strips surrounding whitespace" do + expect(described_class.normalize(" en ")).to eq("en") + expect(described_class.normalize(" jp ")).to eq("ja") + end + + it "is a pure function — does not warn or emit any side effect" do + expect { described_class.normalize("jp") }.not_to output.to_stderr + end + + it "is idempotent — normalizing an already-normalized code is a no-op" do + first = described_class.normalize("jp") + second = described_class.normalize(first) + expect(second).to eq(first) + end + end + + describe ".alias?" do + it "is true for codes in the alias table" do + expect(described_class.alias?("jp")).to be(true) + end + + it "is false for ISO 639-1 codes" do + expect(described_class.alias?("en")).to be(false) + expect(described_class.alias?("ja")).to be(false) + end + + it "is false for nil and empty" do + expect(described_class.alias?(nil)).to be(false) + expect(described_class.alias?("")).to be(false) + end + end + + describe "ALIASES" do + it "is frozen" do + expect(described_class::ALIASES).to be_frozen + end + + it "maps every key to an ISO 639-1 value" do + described_class::ALIASES.each do |original, normalized| + expect(original).to match(/\A[a-z]{2}\z/) + expect(normalized).to match(/\A[a-z]{2}\z/) + expect(original).not_to eq(normalized) + end + end + end +end \ No newline at end of file diff --git a/spec/parcel/sheet_schema_spec.rb b/spec/parcel/sheet_schema_spec.rb index cbe1c4d..5430d9b 100644 --- a/spec/parcel/sheet_schema_spec.rb +++ b/spec/parcel/sheet_schema_spec.rb @@ -49,5 +49,75 @@ s = described_class.from_header_rows(rows) expect(s.find_by_property_id("MDC_P004.fr").name("fr")).to eq("Nom préféré") end + + it "normalizes non-conformant language codes (jp → ja) in column IDs" do + rows = [ + ["#PROPERTY_ID", "MDC_P004_1.en", "MDC_P004_1.jp"], + ["#PROPERTY_NAME.en", "Preferred name", nil], + ["#PROPERTY_NAME.jp", nil, "推奨名"], + ["#DATATYPE", "STRING_TYPE", "TRANSLATABLE_STRING_TYPE"], + ["#REQUIREMENT", "MAND", "MAND"], + ] + s = described_class.from_header_rows(rows) + expect(s.columns.map(&:property_id)) + .to eq(["MDC_P004.en", "MDC_P004.ja"]) + col = s.find_by_property_id("MDC_P004.ja") + expect(col.name("ja")).to eq("推奨名") + end + end + + describe ".canonical_id" do + it "normalizes jp suffix to ja" do + expect(described_class.canonical_id("MDC_P004.jp")).to eq("MDC_P004.ja") + end + + it "preserves standard language suffixes" do + expect(described_class.canonical_id("MDC_P004.de")).to eq("MDC_P004.de") + end + end + + describe "#normalized_language_codes" do + subject(:schema) { described_class.from_header_rows(header_rows) } + + it "is empty when the source spoke ISO 639-1" do + expect(schema.normalized_language_codes).to eq({}) + end + + it "records every non-conformant language code seen in directive keys" do + rows = [ + ["#PROPERTY_ID", "MDC_P004_1.en"], + ["#PROPERTY_NAME.en", "Preferred name"], + ["#PROPERTY_NAME.jp", "推奨名"], + ["#DATATYPE", "TRANSLATABLE_STRING_TYPE"], + ["#REQUIREMENT", "MAND"], + ] + s = described_class.from_header_rows(rows) + expect(s.normalized_language_codes).to eq("jp" => "ja") + end + + it "records non-conformant codes seen in PROPERTY_ID values" do + rows = [ + ["#PROPERTY_ID", "MDC_P004_1.jp"], + ["#PROPERTY_NAME", "Preferred name"], + ["#DATATYPE", "TRANSLATABLE_STRING_TYPE"], + ["#REQUIREMENT", "MAND"], + ] + s = described_class.from_header_rows(rows) + expect(s.normalized_language_codes).to eq("jp" => "ja") + end + + it "returns a frozen hash" do + expect(schema.normalized_language_codes).to be_frozen + end + + it "dedupes — a code seen 1000 times appears once in the audit" do + rows = [["#PROPERTY_ID", "MDC_P004_1.jp"]] + rows.concat(1000.times.map { |i| ["#PROPERTY_NAME.jp", "name #{i}"] }) + rows << ["#DATATYPE", "TRANSLATABLE_STRING_TYPE"] + rows << ["#REQUIREMENT", "MAND"] + s = described_class.from_header_rows(rows) + expect(s.normalized_language_codes.size).to eq(1) + expect(s.normalized_language_codes).to eq("jp" => "ja") + end end end