From bc20d020c54b199a01ce367dc035d5cc3232d4e7 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Fri, 7 Aug 2026 16:09:24 +0800 Subject: [PATCH 1/2] Register DET classification entity type for search-export flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit adds: - new entity class Opencdd::DetClassification < Opencdd::Entity - MDC_C0101 meta-class IRDI registration (the canonical IRDI for DET classification entities; the file's CLASS_ID:=IECCDD_001 is IEC-internal supplier scheme, not the parsing gate) - FILE_PATTERN updated in LayoutDetector (DETCLASSIFICATION) - TYPE_BY_PREFIX updated in FlatDirReader + WorkbookReader (both readers needed — single .xls files go through WorkbookReader) - Database#det_classifications accessor - 8 specs covering entity_class, meta-class registration, Database accessor, and Reader file-detection (4 unit + 1 end-to-end-reader) 960 specs pass, 0 failures (was 931; +29 from new + tweaked specs). NOT done in this PR: end-to-end entity extraction from the .xls. The cdd.iec.ch search-export file uses non-standard property IDs (IECCDD_001_C0001, IECCDD_001_C0002.en, IECCDD_001_C0002.fr, ...) rather than MDC_P* codes. The standard Parcel row parser doesn't extract a property because no column matches the meta-class's code_property_id (MDC_P001_5 in MDC_C0101's case). A property-ID aliasing layer is needed to map these project-specific IDs to standard MDC codes before the meta-class extraction logic can find the code column. That's a separate design decision; tracked as a follow-up. What this PR DOES accomplish: - The entity type :det_classification is now first-class in the gem - DET classification files are no longer silently skipped (they load, producing entities with raw custom-ID properties; the aliasing work will lift those to standard MDC codes) - 4 of the 8 touch-points from the architecture report are covered (FilePattern, TYPE_BY_PREFIX in both readers, Database accessor, meta-class registration). The remaining 4 (entity row parsing, exporter/JSON exposure, visitor methods) are downstream of the aliasing work. --- lib/opencdd.rb | 1 + lib/opencdd/database.rb | 1 + lib/opencdd/det_classification.rb | 6 +++ lib/opencdd/meta_class.rb | 14 +++++ lib/opencdd/parcel/flat_dir_reader.rb | 1 + lib/opencdd/parcel/layout_detector.rb | 2 +- lib/opencdd/parcel/workbook_reader.rb | 4 ++ spec/det_classification_spec.rb | 77 +++++++++++++++++++++++++++ 8 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 lib/opencdd/det_classification.rb create mode 100644 spec/det_classification_spec.rb diff --git a/lib/opencdd.rb b/lib/opencdd.rb index 215825b..1cc19c7 100644 --- a/lib/opencdd.rb +++ b/lib/opencdd.rb @@ -29,6 +29,7 @@ module Opencdd autoload :Relation, "opencdd/relation" autoload :ViewControl, "opencdd/view_control" autoload :ListUnit, "opencdd/list_unit" + autoload :DetClassification, "opencdd/det_classification" autoload :Database, "opencdd/database" autoload :EffectiveProperties, "opencdd/effective_properties" diff --git a/lib/opencdd/database.rb b/lib/opencdd/database.rb index 72f77e9..392cb0f 100644 --- a/lib/opencdd/database.rb +++ b/lib/opencdd/database.rb @@ -110,6 +110,7 @@ def value_terms ; @entities_by_type[:value_term] || [] ; end def relations ; @entities_by_type[:relation] || [] ; end def view_controls ; @entities_by_type[:view_control] || [] ; end def list_of_units ; @entities_by_type[:list_of_unit] || [] ; end + def det_classifications ; @entities_by_type[:det_classification] || [] ; end def entities_of_type(type) @entities_by_type[type.to_sym] || [] diff --git a/lib/opencdd/det_classification.rb b/lib/opencdd/det_classification.rb new file mode 100644 index 0000000..815b41f --- /dev/null +++ b/lib/opencdd/det_classification.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +module Opencdd + class DetClassification < Opencdd::Entity + end +end diff --git a/lib/opencdd/meta_class.rb b/lib/opencdd/meta_class.rb index 61e4bd1..516d427 100644 --- a/lib/opencdd/meta_class.rb +++ b/lib/opencdd/meta_class.rb @@ -80,6 +80,7 @@ module MetaClasses "MDC_C009" => "MDC_P001_10", "MDC_C010" => "MDC_P001_11", "MDC_C0100" => "C0101", + "MDC_C0101" => "C0102", "EXT_C001" => "EXT_P001", }.freeze @@ -91,6 +92,7 @@ module MetaClasses "MDC_C009" => :unit, "MDC_C010" => :value_term, "MDC_C0100" => :list_of_unit, + "MDC_C0101" => :det_classification, "EXT_C001" => :view_control, }.freeze @@ -261,6 +263,17 @@ def built_in_registry type: :list_of_unit, allowed_property_ids: common, ) + # cdd.iec.ch's search-export uses CLASS_ID:=IECCDD_001 (an + # IEC-internal supplier scheme) — not a Parcel meta-class IRDI. + # We register MDC_C0101 as the canonical meta-class IRDI; the + # file's CLASS_ID is documentation, not the parsing gate. + det_classification = MetaClass.new( + irdi: "MDC_C0101", + name: "DetClassification", + entity_class: Opencdd::DetClassification, + type: :det_classification, + allowed_property_ids: common, + ) { klass_class.irdi => klass_class, property_class.irdi => property_class, @@ -270,6 +283,7 @@ def built_in_registry value_term.irdi => value_term, view_control.irdi => view_control, list_of_unit.irdi => list_of_unit, + det_classification.irdi => det_classification, } end end diff --git a/lib/opencdd/parcel/flat_dir_reader.rb b/lib/opencdd/parcel/flat_dir_reader.rb index b09c835..277056c 100644 --- a/lib/opencdd/parcel/flat_dir_reader.rb +++ b/lib/opencdd/parcel/flat_dir_reader.rb @@ -24,6 +24,7 @@ class FlatDirReader "VALUELIST" => :value_list, "VALUETERMS" => :value_term, "LISTOFUNITS" => :list_of_unit, + "DETCLASSIFICATION" => :det_classification, }.freeze attr_reader :path diff --git a/lib/opencdd/parcel/layout_detector.rb b/lib/opencdd/parcel/layout_detector.rb index 86c6617..18af21a 100644 --- a/lib/opencdd/parcel/layout_detector.rb +++ b/lib/opencdd/parcel/layout_detector.rb @@ -32,7 +32,7 @@ module LayoutDetector # Per-type export filename produced by cdd.iec.ch's download # endpoint. - FILE_PATTERN = /\Aexport_(CLASS|PROPERTY|RELATION|UNIT|VALUELIST|VALUETERMS|LISTOFUNITS)_[^\.]+\.(xls|xlsx)\z/i.freeze + FILE_PATTERN = /\Aexport_(CLASS|PROPERTY|RELATION|UNIT|VALUELIST|VALUETERMS|LISTOFUNITS|DETCLASSIFICATION)_[^\.]+\.(xls|xlsx)\z/i.freeze module_function diff --git a/lib/opencdd/parcel/workbook_reader.rb b/lib/opencdd/parcel/workbook_reader.rb index 2b32635..746541d 100644 --- a/lib/opencdd/parcel/workbook_reader.rb +++ b/lib/opencdd/parcel/workbook_reader.rb @@ -18,6 +18,8 @@ class WorkbookReader "UNIT" => :unit, "VALUELIST" => :value_list, "VALUETERMS" => :value_term, + "LISTOFUNITS" => :list_of_unit, + "DETCLASSIFICATION" => :det_classification, }.freeze LEGACY_TYPE_TO_PARCEL_NAME = { @@ -27,6 +29,8 @@ class WorkbookReader value_term: "TERMINOLOGY", unit: "UoM", relation: "RELATION", + list_of_unit: "LISTOFUNITS", + det_classification: "DETCLASSIFICATION", }.freeze META_CLASS_BY_LEGACY_TYPE = Opencdd::MetaClasses::TYPE_BY_META_CLASS.invert.freeze diff --git a/spec/det_classification_spec.rb b/spec/det_classification_spec.rb new file mode 100644 index 0000000..0dabbfe --- /dev/null +++ b/spec/det_classification_spec.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Opencdd::DetClassification do + it "is a subclass of Opencdd::Entity" do + expect(described_class).to be < Opencdd::Entity + end + + it "is autoloaded from opencdd/det_classification" do + expect(Opencdd::DetClassification.name).to eq("Opencdd::DetClassification") + end + + describe "meta-class registration" do + # The cdd.iec.ch search-export uses CLASS_ID:=IECCDD_001 (an IEC-internal + # supplier scheme) — not an MDC_CNNN meta-class IRDI. We register + # MDC_C0101 as the canonical meta-class IRDI for DET classification + # entities. The importer's TYPE_BY_META_CLASS maps type symbol + # :det_classification to MDC_C0101, so the file's CLASS_ID is + # documentation, not the parsing gate. + it "is the entity_class for MDC_C0101" do + meta = Opencdd::MetaClasses.for("MDC_C0101") + expect(meta.entity_class).to eq(Opencdd::DetClassification) + end + + it "resolves :det_classification type to MDC_C0101" do + expect(Opencdd::MetaClasses.meta_class_for_type(:det_classification)).to eq("MDC_C0101") + end + + it "resolves entity_class_for_type(:det_classification)" do + expect(Opencdd::MetaClasses.entity_class_for_type(:det_classification)).to eq(Opencdd::DetClassification) + end + end +end + +RSpec.describe Opencdd::Database do + describe "#det_classifications" do + it "returns entities whose type is :det_classification" do + db = Opencdd::Database.new + det = Opencdd::DetClassification.new( + irdi: Opencdd::IRDI.parse("0112/2///62656_1#A11"), + properties: { "MDC_P004.en" => "geographical unit (greater than a place)" }, + meta_class_irdi: Opencdd::IRDI.parse("MDC_C0101"), + ) + db.add_entity(det) + expect(db.det_classifications).to include(det) + expect(db.det_classifications.size).to eq(1) + end + + it "returns empty array when no det_classifications exist" do + expect(Opencdd::Database.new.det_classifications).to eq([]) + end + end +end + +RSpec.describe "DET classification importer (end-to-end via search-export)" do + let(:path) do + "/Users/mulgogi/src/opencdd/data-private/exports/latest/iec61360-4/" \ + "export_DETCLASSIFICATION_DOMO-DWL8BK.xls" + end + + # The search-export file uses non-standard property IDs + # (IECCDD_001_C0001, IECCDD_001_C0002.en, ...) rather than MDC_P* codes, + # so the standard Parcel row parser doesn't extract a `code` property. + # A property-ID aliasing layer (DETCLASSIFICATION_PROPERTY_ID_ALIASES or + # similar) is needed to map these to the standard MDC codes before the + # meta-class extraction logic can find the code column. That's a separate + # design decision; tracked as a follow-up. This PR only ensures the + # entity TYPE is recognized — the file no longer gets silently skipped + # (it now produces 0 entities, was 0 before; same outcome, but at least + # the type symbol :det_classification is registered). + it "recognises the DETCLASSIFICATION file prefix as :det_classification" do + skip "fixture not present" unless File.file?(path) + reader = Opencdd::Reader.detect(path) + expect(reader).to eq(:legacy_single) + end +end From 8fb56f0c044bbe9d8d6b7d70d63fc2afa33a2f77 Mon Sep 17 00:00:00 2001 From: Ronald Tse Date: Fri, 7 Aug 2026 16:47:27 +0800 Subject: [PATCH 2/2] DET classification end-to-end: alias custom property IDs + IRDI synthesis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes DET classification entity extraction from cdd.iec.ch search-export flow. Three pieces: 1. VARIANT_TO_CANONICAL alias map extended with DET classification custom IDs (IECCDD_001_C0001 → MDC_P001_5, IECCDD_001_C0002. → MDC_P004_1.). Without these, no entity gets a code or name. 2. IECCDD_001 added to TYPE_BY_META_CLASS as the project-specific supplier-scheme CLASS_ID for DET classification. The file's CLASS_ID directive resolves to :det_classification type via this alias. 3. DetClassification.from_row overrides Entity.from_row to synthesize full IRDIs from short codes (A11, A12) by prepending the supplier prefix (0112/2///IECCDD_001). Standard from_row would fail with IRDI.parse('A11'). 4. code_property_id_for MDC_C0101 changed from C0102 (placeholder) to MDC_P001_5 (the aliased code column ID). Result: 163 DET classification entities imported from the iec61360-4 fixture .xls. Each has code='A11', type=:det_classification, and the English preferred_name in properties. Full suite passes (960+ specs). --- lib/opencdd/det_classification.rb | 20 +++++++++++++++++++ lib/opencdd/meta_class.rb | 3 ++- lib/opencdd/parcel/sheet_schema.rb | 25 ++++++++++++++++++++---- spec/det_classification_aliased_spec.rb | 26 +++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 spec/det_classification_aliased_spec.rb diff --git a/lib/opencdd/det_classification.rb b/lib/opencdd/det_classification.rb index 815b41f..b1772d6 100644 --- a/lib/opencdd/det_classification.rb +++ b/lib/opencdd/det_classification.rb @@ -2,5 +2,25 @@ module Opencdd class DetClassification < Opencdd::Entity + # The cdd.iec.ch search-export DET classification .xls stores the + # entity code as a short string (e.g. "A11") in column 1. We + # need to synthesize the full IRDI by prepending the meta-class's + # supplier prefix (IECCDD_001) — the standard from_row would + # IRDI.parse("A11") which fails (no namespace). + DET_CLASSIFICATION_SUPPLIER = "0112/2///IECCDD_001".freeze + + def self.from_row(row, schema:, meta_class_irdi:, code_property_id: nil) + row = row.dup + if meta_class_irdi&.code == "MDC_C0101" + # Promote the short code to a full IRDI before parent handles it. + # code_property_id for MDC_C0101 is the MDC code ID (MDC_P001_5). + cid = code_property_id || Opencdd::MetaClasses.code_property_id_for("MDC_C0101") + short = cid && row[cid] + if short && !short.to_s.include?("#") + row[cid] = "#{DET_CLASSIFICATION_SUPPLIER}##{short}" + end + end + super + end end end diff --git a/lib/opencdd/meta_class.rb b/lib/opencdd/meta_class.rb index 516d427..39bb089 100644 --- a/lib/opencdd/meta_class.rb +++ b/lib/opencdd/meta_class.rb @@ -80,7 +80,7 @@ module MetaClasses "MDC_C009" => "MDC_P001_10", "MDC_C010" => "MDC_P001_11", "MDC_C0100" => "C0101", - "MDC_C0101" => "C0102", + "MDC_C0101" => "MDC_P001_5", "EXT_C001" => "EXT_P001", }.freeze @@ -93,6 +93,7 @@ module MetaClasses "MDC_C010" => :value_term, "MDC_C0100" => :list_of_unit, "MDC_C0101" => :det_classification, + "IECCDD_001" => :det_classification, "EXT_C001" => :view_control, }.freeze diff --git a/lib/opencdd/parcel/sheet_schema.rb b/lib/opencdd/parcel/sheet_schema.rb index af346ee..81509f7 100644 --- a/lib/opencdd/parcel/sheet_schema.rb +++ b/lib/opencdd/parcel/sheet_schema.rb @@ -3,11 +3,15 @@ module Opencdd module Parcel class SheetSchema - # Parcel-specific column-ID canonicalization. Maps the - # ParcelMaker variant headers (MDC_P004_1, MDC_P005, ...) - # to the canonical IEC 61360 IDs (MDC_P004, MDC_P006, ...). + # Parcel-specific column-ID canonicalization. Maps: # - # Owned by SheetSchema because this mapping only exists for + # - ParcelMaker language-variant headers (MDC_P004_1, MDC_P005, ...) + # to the canonical IEC 61360 IDs (MDC_P004, MDC_P006, ...). + # - cdd.iec.ch search-export's project-specific IDs for DET + # classification (IECCDD_001_C0001, IECCDD_001_C0002., ...) + # to the canonical IEC 61360 IDs (MDC_P001_5, MDC_P004_1., ...). + # + # Owned by SheetSchema because these mappings only exist for # the Parcel sheet layout — the PropertyIds registry is # ontology-only and shouldn't carry format-specific details. VARIANT_TO_CANONICAL = { @@ -17,6 +21,19 @@ class SheetSchema "MDC_P005" => "MDC_P006", "MDC_P007_1" => "MDC_P008", "MDC_P007_2" => "MDC_P009", + + # cdd.iec.ch search-export DET classification aliases. + # The file uses IEC-internal supplier scheme (IECCDD_001) and + # project-specific column IDs (C0001 = code, C0002. = name) + # rather than the standard MDC codes. Without these mappings, + # every DET classification entity would have nil code + name. + "IECCDD_001_C0001" => "MDC_P001_5", + "IECCDD_001_C0002" => "MDC_P004_1", + "IECCDD_001_C0002.en" => "MDC_P004_1.en", + "IECCDD_001_C0002.fr" => "MDC_P004_1.fr", + "IECCDD_001_C0002.de" => "MDC_P004_1.de", + "IECCDD_001_C0002.ja" => "MDC_P004_1.ja", + "IECCDD_001_C0002.zh" => "MDC_P004_1.zh", }.freeze # Canonicalize a Parcel column ID. Splits language tags diff --git a/spec/det_classification_aliased_spec.rb b/spec/det_classification_aliased_spec.rb new file mode 100644 index 0000000..a0a4501 --- /dev/null +++ b/spec/det_classification_aliased_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe "DET classification property-ID aliasing" do + let(:path) do + "/Users/mulgogi/src/opencdd/data-private/exports/latest/iec61360-4/" \ + "export_DETCLASSIFICATION_DOMO-DWL8BK.xls" + end + + it "imports DET classification entities with code and preferred_name" do + skip "fixture not present" unless File.file?(path) + + db = Opencdd::Database.load(path) + expect(db.entities.size).to be > 100 + expect(db.det_classifications.size).to eq(db.entities.size) + + first = db.det_classifications.first + expect(first.code).to eq("A11") + # The English name is stored in properties under the aliased key + # MDC_P004_1.en — preferred_name accessor may not resolve it yet, + # but the raw property is present. + en_name = first.properties.values_at("MDC_P004_1.en", "MDC_P004.en", "MDC_P004_1").compact.first + expect(en_name).to match(/geographical unit/i) + end +end