From 1ac46658c1473d6c39255f60095ada534f2ac0f3 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Fri, 7 Aug 2026 14:31:45 +0800 Subject: [PATCH 1/2] Normalize non-standard language codes to ISO 639-1 at XLS ingestion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IEC CDD source .xls files use "jp" for Japanese instead of the ISO 639-1 code "ja". The gem previously passed these through verbatim, creating a mismatch with the browser's CSS visibility rules and the LanguageSwitcher, which expect ISO codes. Add Languages.normalize(code) — a class method that maps known non-conformant codes to their ISO equivalents and emits a warning on stderr so data-quality issues are visible. The LANG_ALIASES map is extensible for future non-standard codes. Apply normalization at the two points where language codes enter from the XLS source: 1. SheetSchema.canonical_id — column IDs like MDC_P004.jp are now canonicalized to MDC_P004.ja before entering the entity model. 2. SheetSchema#lang_hash_for — directive-row language keys (PROPERTY_NAME.jp etc.) are normalized before building the per-column name/definition/note language hashes. Also apply normalization in Languages#initialize and Languages.from_properties so any code path that constructs a Languages object gets clean codes. The warning fires once per non-conformant code encountered, making it easy to audit which dictionaries still carry legacy codes. --- lib/opencdd/languages.rb | 35 +++++++++++++++++++--- lib/opencdd/parcel/sheet_schema.rb | 5 +++- spec/languages_spec.rb | 48 ++++++++++++++++++++++++++++++ spec/parcel/sheet_schema_spec.rb | 25 ++++++++++++++++ 4 files changed, 108 insertions(+), 5 deletions(-) diff --git a/lib/opencdd/languages.rb b/lib/opencdd/languages.rb index 6d85dc1..1c27533 100644 --- a/lib/opencdd/languages.rb +++ b/lib/opencdd/languages.rb @@ -13,11 +13,21 @@ module Opencdd # to render a language switcher and by the data pipeline to report # language coverage. class Languages + # IEC CDD source .xls files sometimes use non-standard language + # codes that diverge from ISO 639-1. Map them here so the entire + # ecosystem (gem, JSON wire format, browser CSS, TS model) speaks + # the same ISO code. When a non-conformant code is encountered, + # +normalize+ emits a one-line warning on stderr so data-quality + # issues are visible without silently rewriting. + LANG_ALIASES = { + "jp" => "ja", + }.freeze + attr_reader :source, :translations def initialize(source: "en", translations: []) - @source = source.to_s - @translations = Array(translations).map(&:to_s).uniq - [@source] + @source = self.class.normalize(source) + @translations = Array(translations).map { |t| self.class.normalize(t) }.uniq - [@source] freeze end @@ -26,7 +36,7 @@ def all end def include?(lang) - all.include?(lang.to_s) + all.include?(self.class.normalize(lang)) end def empty? @@ -51,6 +61,23 @@ def hash [source, translations].hash end + # Normalize a language code to ISO 639-1. + # + # Returns the input unchanged if it is already standard. If the + # code is a known non-conformant alias (e.g. "jp" from IEC CDD + # source .xls), returns the ISO equivalent ("ja") and emits a + # warning on stderr so the data-quality issue is visible. + def self.normalize(lang) + return lang if lang.nil? + code = lang.to_s.strip + return code if code.empty? + if LANG_ALIASES.key?(code) + warn "[opencdd] non-conformant language code #{code.inspect} → #{LANG_ALIASES[code].inspect} (ISO 639-1)" + return LANG_ALIASES[code] + end + code + end + # Scan a properties hash for +.+ keys and # return a Languages object covering every language seen. The # source language defaults to +default_source+ when no explicit @@ -60,7 +87,7 @@ def self.from_properties(properties, default_source: "en") next unless key.include?(".") prefix, lang = key.split(".", 2) next unless lang =~ /\A[a-z]{2}(-[a-z0-9]+)?\z/i - acc << lang + acc << normalize(lang) end source = langs.include?(default_source) ? default_source : (langs.first || default_source) translations = langs.to_a - [source] diff --git a/lib/opencdd/parcel/sheet_schema.rb b/lib/opencdd/parcel/sheet_schema.rb index 25d6a85..0b66c66 100644 --- a/lib/opencdd/parcel/sheet_schema.rb +++ b/lib/opencdd/parcel/sheet_schema.rb @@ -22,6 +22,8 @@ class SheetSchema # Canonicalize a Parcel column ID. Splits language tags # (.) so they round-trip cleanly. Returns the # canonical ID (or the input unchanged if no mapping exists). + # Language codes are normalized to ISO 639-1 via + # +Opencdd::Languages.normalize+. def self.canonical_id(raw_id) return nil if raw_id.nil? s = raw_id.to_s.strip @@ -30,7 +32,7 @@ def self.canonical_id(raw_id) base = match ? match.pre_match : s lang = match && match[:lang] canonical = VARIANT_TO_CANONICAL[base] || base - lang ? "#{canonical}.#{lang}" : canonical + lang ? "#{canonical}.#{Opencdd::Languages.normalize(lang)}" : canonical end DIRECTIVE_ROWS = %w[ @@ -260,6 +262,7 @@ def lang_hash_for(directive, col_idx) next if v.nil? s = v.to_s.strip next if s.empty? + lang = Opencdd::Languages.normalize(lang) if lang && !lang.empty? lang = "en" if lang.nil? || lang.empty? h[lang] = s end diff --git a/spec/languages_spec.rb b/spec/languages_spec.rb index 6f4ba74..9f5b32b 100644 --- a/spec/languages_spec.rb +++ b/spec/languages_spec.rb @@ -83,6 +83,48 @@ end end + describe ".normalize" do + it "passes through ISO 639-1 codes unchanged" do + expect(described_class.normalize("en")).to eq("en") + expect(described_class.normalize("ja")).to eq("ja") + expect(described_class.normalize("de")).to eq("de") + end + + it "maps jp to ja (IEC CDD non-conformant)" do + expect(described_class.normalize("jp")).to eq("ja") + end + + it "warns on stderr when a non-conformant code is seen" do + expect { described_class.normalize("jp") } + .to output(/non-conformant language code "jp"/).to_stderr + end + + it "does not warn for standard codes" do + expect { described_class.normalize("en") }.not_to output.to_stderr + end + + it "handles nil and empty gracefully" do + expect(described_class.normalize(nil)).to be_nil + expect(described_class.normalize("")).to eq("") + end + + it "strips whitespace" do + expect(described_class.normalize(" en ")).to eq("en") + end + end + + describe "normalization on construction" do + it "normalizes source and translations" do + langs = described_class.new(source: "en", translations: %w[jp fr]) + expect(langs.translations).to eq(%w[ja fr]) + end + + it "does not double-count jp and ja" do + langs = described_class.new(source: "en", translations: %w[jp ja]) + expect(langs.translations).to eq(%w[ja]) + end + end + describe ".from_properties" do it "scans properties hash for language-tagged keys" do props = { "MDC_P004.en" => "Vehicle", "MDC_P004.fr" => "Véhicule", "MDC_P004.de" => "Fahrzeug" } @@ -102,5 +144,11 @@ langs = described_class.from_properties(props) expect(langs.all).to eq(%w[en]) end + + it "normalizes non-conformant codes like jp to ja" do + props = { "MDC_P004.en" => "Vehicle", "MDC_P004.jp" => "車両" } + langs = described_class.from_properties(props) + expect(langs.translations).to eq(%w[ja]) + end end end diff --git a/spec/parcel/sheet_schema_spec.rb b/spec/parcel/sheet_schema_spec.rb index cbe1c4d..cd5b670 100644 --- a/spec/parcel/sheet_schema_spec.rb +++ b/spec/parcel/sheet_schema_spec.rb @@ -49,5 +49,30 @@ s = described_class.from_header_rows(rows) expect(s.find_by_property_id("MDC_P004.fr").name("fr")).to eq("Nom préféré") end + + it "normalizes non-conformant language codes (jp → ja) in column IDs" do + rows = [ + ["#PROPERTY_ID", "MDC_P004_1.en", "MDC_P004_1.jp"], + ["#PROPERTY_NAME.en", "Preferred name", nil], + ["#PROPERTY_NAME.jp", nil, "推奨名"], + ["#DATATYPE", "STRING_TYPE", "TRANSLATABLE_STRING_TYPE"], + ["#REQUIREMENT", "MAND", "MAND"], + ] + s = described_class.from_header_rows(rows) + expect(s.columns.map(&:property_id)) + .to eq(["MDC_P004.en", "MDC_P004.ja"]) + col = s.find_by_property_id("MDC_P004.ja") + expect(col.name("ja")).to eq("推奨名") + end + end + + describe ".canonical_id" do + it "normalizes jp suffix to ja" do + expect(described_class.canonical_id("MDC_P004.jp")).to eq("MDC_P004.ja") + end + + it "preserves standard language suffixes" do + expect(described_class.canonical_id("MDC_P004.de")).to eq("MDC_P004.de") + end end end From 55a86294c308e2453285521c4f939590f7b63a08 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Fri, 7 Aug 2026 15:54:15 +0800 Subject: [PATCH 2/2] Apply audit Option A: extract language aliases to Parcel layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original PR landed language normalization inside Opencdd::Languages (the model class), where it had three architectural problems: - emitted warn on stderr per call (log spam in batch ingestion), - coupled a model object to a source-format concern, - and applied the same rewrite at five different sites. This commit applies Option A from the audit: - Opencdd::Parcel::LanguageAliases owns the jp→ja alias table as a Parcel-format concern. The aliases are an artefact of one specific source format (IEC CDD .xls exports), not of the canonical CDD model. - SheetSchema applies normalization at the ingestion boundary (canonical_id + lang_hash_for); the per-call stderr warn is removed, eliminating log spam in batch builds. - SheetSchema exposes #normalized_language_codes (frozen Hash {original => normalized}) for structured data-quality reporting -- replaces the noisy warn with a per-schema audit trail. - Opencdd::Languages is now a pure value object: it assumes ISO 639-1 codes and does not rewrite at construction or query time. Source-format quirks stay at the format layer. - Drop redundant nil/empty guard in SheetSchema#lang_hash_for. - Use `_` for the unused prefix destructure in Languages.from_properties. Specs: new spec/parcel/language_aliases_spec covers the pure mapping; spec/parcel/sheet_schema_spec adds #normalized_language_codes coverage. spec/languages_spec drops .normalize blocks and adds a "model contract" describe documenting that Languages does not rewrite. Closes #23. --- lib/opencdd/languages.rb | 45 +++++------------ lib/opencdd/parcel.rb | 1 + lib/opencdd/parcel/language_aliases.rb | 42 ++++++++++++++++ lib/opencdd/parcel/sheet_schema.rb | 31 ++++++++++-- spec/languages_spec.rb | 67 ++++++++------------------ spec/parcel/language_aliases_spec.rb | 67 ++++++++++++++++++++++++++ spec/parcel/sheet_schema_spec.rb | 45 +++++++++++++++++ 7 files changed, 215 insertions(+), 83 deletions(-) create mode 100644 lib/opencdd/parcel/language_aliases.rb create mode 100644 spec/parcel/language_aliases_spec.rb diff --git a/lib/opencdd/languages.rb b/lib/opencdd/languages.rb index 1c27533..fa8f4ae 100644 --- a/lib/opencdd/languages.rb +++ b/lib/opencdd/languages.rb @@ -12,22 +12,18 @@ module Opencdd # "what languages does this dictionary have?" — used by the browser # to render a language switcher and by the data pipeline to report # language coverage. + # + # This is a model object. It does not normalize language codes: + # source-format quirks (e.g. IEC CDD .xls files using "jp" for + # Japanese) are normalized at the SheetSchema ingestion boundary + # via +Opencdd::Parcel::LanguageAliases+, so every consumer of + # this class can assume ISO 639-1 codes unconditionally. class Languages - # IEC CDD source .xls files sometimes use non-standard language - # codes that diverge from ISO 639-1. Map them here so the entire - # ecosystem (gem, JSON wire format, browser CSS, TS model) speaks - # the same ISO code. When a non-conformant code is encountered, - # +normalize+ emits a one-line warning on stderr so data-quality - # issues are visible without silently rewriting. - LANG_ALIASES = { - "jp" => "ja", - }.freeze - attr_reader :source, :translations def initialize(source: "en", translations: []) - @source = self.class.normalize(source) - @translations = Array(translations).map { |t| self.class.normalize(t) }.uniq - [@source] + @source = source.to_s + @translations = Array(translations).map(&:to_s).uniq - [@source] freeze end @@ -36,7 +32,7 @@ def all end def include?(lang) - all.include?(self.class.normalize(lang)) + all.include?(lang.to_s) end def empty? @@ -61,23 +57,6 @@ def hash [source, translations].hash end - # Normalize a language code to ISO 639-1. - # - # Returns the input unchanged if it is already standard. If the - # code is a known non-conformant alias (e.g. "jp" from IEC CDD - # source .xls), returns the ISO equivalent ("ja") and emits a - # warning on stderr so the data-quality issue is visible. - def self.normalize(lang) - return lang if lang.nil? - code = lang.to_s.strip - return code if code.empty? - if LANG_ALIASES.key?(code) - warn "[opencdd] non-conformant language code #{code.inspect} → #{LANG_ALIASES[code].inspect} (ISO 639-1)" - return LANG_ALIASES[code] - end - code - end - # Scan a properties hash for +.+ keys and # return a Languages object covering every language seen. The # source language defaults to +default_source+ when no explicit @@ -85,13 +64,13 @@ def self.normalize(lang) def self.from_properties(properties, default_source: "en") langs = properties.keys.each_with_object(Set.new) do |key, acc| next unless key.include?(".") - prefix, lang = key.split(".", 2) + _, lang = key.split(".", 2) next unless lang =~ /\A[a-z]{2}(-[a-z0-9]+)?\z/i - acc << normalize(lang) + acc << lang end source = langs.include?(default_source) ? default_source : (langs.first || default_source) translations = langs.to_a - [source] new(source: source, translations: translations) end end -end +end \ No newline at end of file diff --git a/lib/opencdd/parcel.rb b/lib/opencdd/parcel.rb index 69e945d..2c6a910 100644 --- a/lib/opencdd/parcel.rb +++ b/lib/opencdd/parcel.rb @@ -7,6 +7,7 @@ module Parcel autoload :Metadata, "opencdd/parcel/metadata" autoload :SheetSchema, "opencdd/parcel/sheet_schema" + autoload :LanguageAliases, "opencdd/parcel/language_aliases" autoload :Sheet, "opencdd/parcel/sheet" autoload :Workbook, "opencdd/parcel/workbook" autoload :WorkbookReader, "opencdd/parcel/workbook_reader" diff --git a/lib/opencdd/parcel/language_aliases.rb b/lib/opencdd/parcel/language_aliases.rb new file mode 100644 index 0000000..ea3c492 --- /dev/null +++ b/lib/opencdd/parcel/language_aliases.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Opencdd + module Parcel + # Maps non-conformant language codes found in IEC CDD source .xls + # exports to their ISO 639-1 equivalents. + # + # This is a Parcel-layer concern, not a model concern: the aliases + # exist because one specific source format (IEC CDD XLS) uses + # non-standard codes in its property ID suffixes and directive + # rows. The +Languages+ value object — and every downstream + # consumer (JSON wire format, browser, TS model) — speaks ISO + # 639-1 unconditionally. + # + # The normalization happens at the SheetSchema ingestion + # boundary so downstream code never sees an alias. SheetSchema + # also exposes the set of original-to-canonical mappings it + # applied via +#normalized_language_codes+ for data-quality + # reporting. + module LanguageAliases + ALIASES = { + "jp" => "ja", + }.freeze + + # Returns the ISO 639-1 form of +code+, or the input unchanged + # if it is already standard. Pure function: no I/O, no side + # effects. + def self.normalize(code) + return code if code.nil? + s = code.to_s.strip + return s if s.empty? + ALIASES.fetch(s, s) + end + + # True when +code+ would be rewritten by +normalize+. + def self.alias?(code) + return false if code.nil? + ALIASES.key?(code.to_s.strip) + end + end + end +end \ No newline at end of file diff --git a/lib/opencdd/parcel/sheet_schema.rb b/lib/opencdd/parcel/sheet_schema.rb index 0b66c66..af346ee 100644 --- a/lib/opencdd/parcel/sheet_schema.rb +++ b/lib/opencdd/parcel/sheet_schema.rb @@ -23,7 +23,7 @@ class SheetSchema # (.) so they round-trip cleanly. Returns the # canonical ID (or the input unchanged if no mapping exists). # Language codes are normalized to ISO 639-1 via - # +Opencdd::Languages.normalize+. + # +Opencdd::Parcel::LanguageAliases.normalize+. def self.canonical_id(raw_id) return nil if raw_id.nil? s = raw_id.to_s.strip @@ -32,7 +32,7 @@ def self.canonical_id(raw_id) base = match ? match.pre_match : s lang = match && match[:lang] canonical = VARIANT_TO_CANONICAL[base] || base - lang ? "#{canonical}.#{Opencdd::Languages.normalize(lang)}" : canonical + lang ? "#{canonical}.#{Opencdd::Parcel::LanguageAliases.normalize(lang)}" : canonical end DIRECTIVE_ROWS = %w[ @@ -112,7 +112,7 @@ def obsolete? DIRECTIVE_ROW_PREFIX = "#".freeze - attr_reader :columns, :columns_by_id, :column_directives + attr_reader :columns, :columns_by_id, :column_directives, :normalized_language_codes def initialize @columns = [] @@ -196,6 +196,7 @@ def finalize! @columns_by_id[col.property_id] = col end + @normalized_language_codes = compute_normalized_language_codes.freeze freeze end @@ -262,12 +263,34 @@ def lang_hash_for(directive, col_idx) next if v.nil? s = v.to_s.strip next if s.empty? - lang = Opencdd::Languages.normalize(lang) if lang && !lang.empty? + lang = Opencdd::Parcel::LanguageAliases.normalize(lang) lang = "en" if lang.nil? || lang.empty? h[lang] = s end h end + + # Audit trail of language-code normalizations applied at this + # schema's ingestion boundary. Returns a frozen Hash mapping each + # original (non-conformant) language code seen in the source to + # its ISO 639-1 equivalent. Empty when the source spoke ISO. + def compute_normalized_language_codes + seen = {} + @column_directives.each_key do |key| + _, lang = key.to_s.split(".", 2) + next if lang.nil? || lang.empty? + normalized = Opencdd::Parcel::LanguageAliases.normalize(lang) + seen[lang] = normalized if normalized != lang + end + @columns.each do |col| + next if col.raw_property_id.nil? + _, lang = col.raw_property_id.split(".", 2) + next if lang.nil? || lang.empty? + normalized = Opencdd::Parcel::LanguageAliases.normalize(lang) + seen[lang] = normalized if normalized != lang + end + seen + end end end end diff --git a/spec/languages_spec.rb b/spec/languages_spec.rb index 9f5b32b..83e33da 100644 --- a/spec/languages_spec.rb +++ b/spec/languages_spec.rb @@ -83,48 +83,6 @@ end end - describe ".normalize" do - it "passes through ISO 639-1 codes unchanged" do - expect(described_class.normalize("en")).to eq("en") - expect(described_class.normalize("ja")).to eq("ja") - expect(described_class.normalize("de")).to eq("de") - end - - it "maps jp to ja (IEC CDD non-conformant)" do - expect(described_class.normalize("jp")).to eq("ja") - end - - it "warns on stderr when a non-conformant code is seen" do - expect { described_class.normalize("jp") } - .to output(/non-conformant language code "jp"/).to_stderr - end - - it "does not warn for standard codes" do - expect { described_class.normalize("en") }.not_to output.to_stderr - end - - it "handles nil and empty gracefully" do - expect(described_class.normalize(nil)).to be_nil - expect(described_class.normalize("")).to eq("") - end - - it "strips whitespace" do - expect(described_class.normalize(" en ")).to eq("en") - end - end - - describe "normalization on construction" do - it "normalizes source and translations" do - langs = described_class.new(source: "en", translations: %w[jp fr]) - expect(langs.translations).to eq(%w[ja fr]) - end - - it "does not double-count jp and ja" do - langs = described_class.new(source: "en", translations: %w[jp ja]) - expect(langs.translations).to eq(%w[ja]) - end - end - describe ".from_properties" do it "scans properties hash for language-tagged keys" do props = { "MDC_P004.en" => "Vehicle", "MDC_P004.fr" => "Véhicule", "MDC_P004.de" => "Fahrzeug" } @@ -144,11 +102,28 @@ langs = described_class.from_properties(props) expect(langs.all).to eq(%w[en]) end + end + + describe "model contract" do + it "does not normalize source — callers must supply ISO 639-1 codes" do + # Source-format normalization lives at the ingestion boundary + # (Opencdd::Parcel::SheetSchema via LanguageAliases). By the + # time a Languages value object is built, codes are already + # canonical. Passing "jp" through should NOT be silently + # rewritten — that would hide upstream data-quality issues. + langs = described_class.new(source: "jp", translations: []) + expect(langs.source).to eq("jp") + end - it "normalizes non-conformant codes like jp to ja" do - props = { "MDC_P004.en" => "Vehicle", "MDC_P004.jp" => "車両" } - langs = described_class.from_properties(props) - expect(langs.translations).to eq(%w[ja]) + it "does not normalize translations" do + langs = described_class.new(source: "en", translations: %w[jp]) + expect(langs.translations).to eq(%w[jp]) + end + + it "include? does not rewrite the lookup argument" do + langs = described_class.new(source: "en", translations: %w[ja]) + expect(langs.include?("jp")).to be(false) + expect(langs.include?("ja")).to be(true) end end end diff --git a/spec/parcel/language_aliases_spec.rb b/spec/parcel/language_aliases_spec.rb new file mode 100644 index 0000000..e443a6c --- /dev/null +++ b/spec/parcel/language_aliases_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Opencdd::Parcel::LanguageAliases do + describe ".normalize" do + it "passes through ISO 639-1 codes unchanged" do + expect(described_class.normalize("en")).to eq("en") + expect(described_class.normalize("ja")).to eq("ja") + expect(described_class.normalize("de")).to eq("de") + end + + it "maps jp to ja (IEC CDD non-conformant)" do + expect(described_class.normalize("jp")).to eq("ja") + end + + it "handles nil and empty gracefully" do + expect(described_class.normalize(nil)).to be_nil + expect(described_class.normalize("")).to eq("") + end + + it "strips surrounding whitespace" do + expect(described_class.normalize(" en ")).to eq("en") + expect(described_class.normalize(" jp ")).to eq("ja") + end + + it "is a pure function — does not warn or emit any side effect" do + expect { described_class.normalize("jp") }.not_to output.to_stderr + end + + it "is idempotent — normalizing an already-normalized code is a no-op" do + first = described_class.normalize("jp") + second = described_class.normalize(first) + expect(second).to eq(first) + end + end + + describe ".alias?" do + it "is true for codes in the alias table" do + expect(described_class.alias?("jp")).to be(true) + end + + it "is false for ISO 639-1 codes" do + expect(described_class.alias?("en")).to be(false) + expect(described_class.alias?("ja")).to be(false) + end + + it "is false for nil and empty" do + expect(described_class.alias?(nil)).to be(false) + expect(described_class.alias?("")).to be(false) + end + end + + describe "ALIASES" do + it "is frozen" do + expect(described_class::ALIASES).to be_frozen + end + + it "maps every key to an ISO 639-1 value" do + described_class::ALIASES.each do |original, normalized| + expect(original).to match(/\A[a-z]{2}\z/) + expect(normalized).to match(/\A[a-z]{2}\z/) + expect(original).not_to eq(normalized) + end + end + end +end \ No newline at end of file diff --git a/spec/parcel/sheet_schema_spec.rb b/spec/parcel/sheet_schema_spec.rb index cd5b670..5430d9b 100644 --- a/spec/parcel/sheet_schema_spec.rb +++ b/spec/parcel/sheet_schema_spec.rb @@ -75,4 +75,49 @@ expect(described_class.canonical_id("MDC_P004.de")).to eq("MDC_P004.de") end end + + describe "#normalized_language_codes" do + subject(:schema) { described_class.from_header_rows(header_rows) } + + it "is empty when the source spoke ISO 639-1" do + expect(schema.normalized_language_codes).to eq({}) + end + + it "records every non-conformant language code seen in directive keys" do + rows = [ + ["#PROPERTY_ID", "MDC_P004_1.en"], + ["#PROPERTY_NAME.en", "Preferred name"], + ["#PROPERTY_NAME.jp", "推奨名"], + ["#DATATYPE", "TRANSLATABLE_STRING_TYPE"], + ["#REQUIREMENT", "MAND"], + ] + s = described_class.from_header_rows(rows) + expect(s.normalized_language_codes).to eq("jp" => "ja") + end + + it "records non-conformant codes seen in PROPERTY_ID values" do + rows = [ + ["#PROPERTY_ID", "MDC_P004_1.jp"], + ["#PROPERTY_NAME", "Preferred name"], + ["#DATATYPE", "TRANSLATABLE_STRING_TYPE"], + ["#REQUIREMENT", "MAND"], + ] + s = described_class.from_header_rows(rows) + expect(s.normalized_language_codes).to eq("jp" => "ja") + end + + it "returns a frozen hash" do + expect(schema.normalized_language_codes).to be_frozen + end + + it "dedupes — a code seen 1000 times appears once in the audit" do + rows = [["#PROPERTY_ID", "MDC_P004_1.jp"]] + rows.concat(1000.times.map { |i| ["#PROPERTY_NAME.jp", "name #{i}"] }) + rows << ["#DATATYPE", "TRANSLATABLE_STRING_TYPE"] + rows << ["#REQUIREMENT", "MAND"] + s = described_class.from_header_rows(rows) + expect(s.normalized_language_codes.size).to eq(1) + expect(s.normalized_language_codes).to eq("jp" => "ja") + end + end end