From 9f2831089fd7837c4252a6b9d48cd3e8f61f0cd3 Mon Sep 17 00:00:00 2001 From: krwong Date: Wed, 1 Jul 2026 13:36:37 -0400 Subject: [PATCH 1/8] add aspace_ref_id generate_from_index command, add generateAspaceRefIdMappingFromCdmIndexDb and hasProjectContriDescriAspaceRefIdFields methods, update tests and test resources --- .../migration/cdm/AspaceRefIdCommand.java | 24 +++++ .../cdm/services/AspaceRefIdService.java | 87 +++++++++++++++++++ .../migration/cdm/AspaceRefIdCommandIT.java | 15 ++++ .../cdm/services/AspaceRefIdServiceTest.java | 35 ++++++++ .../03883/index/description/desc.all | 3 + src/test/resources/findingaid_fields.csv | 3 +- 6 files changed, 166 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommand.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommand.java index 9861c197..70f1325e 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommand.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommand.java @@ -83,6 +83,30 @@ public int generateFromCsv() throws Exception { } } + @Command(name = "generate_from_index", + description = {"Generate the optional aspace ref id mapping file for this project " + + "using the indexed ead to cdm metadata file, cdm_index.db.", + "A ref_id_mapping.csv template will be created for this project, " + + "with record ids, hook ids, and aspace ref ids populated."}) + public int generateFromIndex() throws Exception { + long start = System.nanoTime(); + + try { + initialize(); + aspaceRefIdService.generateAspaceRefIdMappingFromCdmIndexDb(); + outputLogger.info("Aspace ref id mapping generated for {} in {}s", project.getProjectName(), + (System.nanoTime() - start) / 1e9); + return 0; + } catch (MigrationException | IllegalArgumentException e) { + outputLogger.info("Cannot generate aspace ref id mapping: {}", e.getMessage()); + return 1; + } catch (Exception e) { + log.error("Failed to generate aspace ref id template", e); + outputLogger.info("Failed to generate aspace ref id template: {}", e.getMessage(), e); + return 1; + } + } + @Command(name = "validate", description = {"Validate the aspace ref id mappings for this project"}) public int validate(@Option(names = { "-f", "--force"}, diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java index 0d9f9c5e..a678b9c8 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java @@ -40,7 +40,9 @@ public class AspaceRefIdService { private CdmIndexService indexService; private Path hookIdRefIdMapPath; private Boolean projectHasContriAndDescri = null; + private Boolean projectHasContriDescriAspaceRefId = null; + public static final String REF_ID_FIELD = "refid"; public static final String[] HOOKID_REFID_CSV_HEADERS = {"cache_hookid", "normalized_cache_hookid", "collid", "ref_id", "ao_title", "tc_type", "tc_indicator", "sc_type", "sc_indicator", "gc_type", "gc_indicator", "aspace_hookid", "cdm_alias"}; @@ -98,6 +100,57 @@ public void generateAspaceRefIdMappingFromHookIdRefIdCsv() throws IOException { setUpdatedDate(Instant.now()); } + /** + * Generate the aspace ref id mapping using the indexed ead to cdm metadata file (cdm_index.db) + * Record ids, hook ids, and ref ids populated + * @throws Exception + */ + public void generateAspaceRefIdMappingFromCdmIndexDb() throws IOException { + assertProjectStateValid(); + + if (!hasProjectContriDescriAspaceRefIdFields()) { + throw new InvalidProjectStateException("Project has no contri field named hook id, " + + "and/or descri field named collection number, and/or refid field named aspace ref id"); + } + + try (BufferedWriter writer = Files.newBufferedWriter(getMappingPath()); + var csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT.withHeader(AspaceRefIdInfo.CSV_HEADERS))) { + String query = "select " + CdmFieldInfo.CDM_ID + "," + FindingAidService.DESCRI_FIELD + "," + + FindingAidService.CONTRI_FIELD + "," + REF_ID_FIELD + + " from " + CdmIndexService.TB_NAME + + " where " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'" + + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_COMPOUND_OBJECT + "'" + + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_DOCUMENT_PDF + "'" + + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null" + + " and " + CdmIndexService.PARENT_ID_FIELD + " is null" + + " and " + FindingAidService.DESCRI_FIELD + " is not null" + + " and " + FindingAidService.CONTRI_FIELD + " is not null"; + + getIndexService(); + try (Connection conn = indexService.openDbConnection()) { + var stmt = conn.createStatement(); + var rs = stmt.executeQuery(query); + while (rs.next()) { + // if dmrecord, descri, contri, and ref_id fields are not blank, + // add dmrecord, descri_contri (hook id when combined), and ref_id to aspace ref id mapping + var dmrecord = rs.getString(1); + var descri = rs.getString(2); + var contri = rs.getString(3); + var refId = rs.getString(4); + if (!dmrecord.isBlank() && !descri.isBlank() && !contri.isBlank() & !refId.isBlank()) { + // remove -v from descri for matching purposes + csvPrinter.printRecord(dmrecord, + descri.replace("-z", "") + "_" + contri, refId); + } + } + } catch (SQLException e) { + throw new MigrationException("Error interacting with export index", e); + } + } + + setUpdatedDate(Instant.now()); + } + protected void setUpdatedDate(Instant timestamp) throws IOException { project.getProjectProperties().setAspaceRefIdMappingsUpdatedDate(timestamp); ProjectPropertiesSerialization.write(project); @@ -219,6 +272,40 @@ private boolean hasProjectContriAndDescriFields() { return projectHasContriAndDescri; } + private boolean hasProjectContriDescriAspaceRefIdFields() { + if (projectHasContriDescriAspaceRefId == null) { + // check if project has contri field named hook id, descri field named collection number, + // and refid field named aspace ref id + boolean hasContri = false; + boolean hasDescri = false; + boolean hasAspaceRefId = false; + fieldService.validateFieldsFile(project); + CdmFieldInfo fieldInfo = fieldService.loadFieldsFromProject(project); + Map fields = fieldInfo.getFields().stream() + .filter(f -> !f.getSkipExport()) + .collect(Collectors.toMap(CdmFieldInfo.CdmFieldEntry::getNickName, + CdmFieldInfo.CdmFieldEntry::getDescription)); + + for (Map.Entry entry : fields.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (key.equalsIgnoreCase(FindingAidService.CONTRI_FIELD) + && value.equalsIgnoreCase(FindingAidService.HOOK_ID_FIELD_DESC)) { + hasContri = true; + } else if (key.equalsIgnoreCase(FindingAidService.DESCRI_FIELD) + && value.equalsIgnoreCase(FindingAidService.COLLECTION_NUMBER_FIELD_DESC)) { + hasDescri = true; + } else if (key.equalsIgnoreCase(REF_ID_FIELD) + && value.equalsIgnoreCase("aspace ref id")) { + hasAspaceRefId = true; + } + } + + projectHasContriDescriAspaceRefId = hasContri && hasDescri && hasAspaceRefId; + } + return projectHasContriDescriAspaceRefId; + } + /** * @return the aspace ref id mapping info for the configured project * @throws IOException diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommandIT.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommandIT.java index bd316d17..d5b6a824 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommandIT.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommandIT.java @@ -97,6 +97,21 @@ public void generateFromCsvAspaceRefIdMappingSucceedsTest() throws Exception { assertUpdatedDatePresent(); } + @Test + public void generateFromIndexAspaceRefIdMappingSucceedsTest() throws Exception { + indexExportSamples(); + + String[] args = new String[] { + "-w", project.getProjectPath().toString(), + "--env-config", chompbConfigPath, + "aspace_ref_id", "generate_from_index"}; + executeExpectSuccess(args); + + assertTrue(Files.exists(project.getAspaceRefIdMappingPath())); + + assertUpdatedDatePresent(); + } + @Test public void validateValidTest() throws Exception { indexExportSamples(); diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java index 8b1c993d..fd36b959 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java @@ -194,6 +194,41 @@ public void generateFromHookIdRefIdCsvNoContriDescriTest() throws Exception { assertTrue(actualMessage.contains(expectedMessage)); } + @Test + public void generateFromCdmIndexDbTest() throws Exception { + testHelper.indexExportData(Paths.get("src/test/resources/findingaid_fields.csv"), "03883"); + service.generateAspaceRefIdMappingFromCdmIndexDb(); + + assertTrue(Files.exists(project.getAspaceRefIdMappingPath())); + try (CSVParser csvParser = parser()) { + List rows = csvParser.getRecords(); + assertEquals("0", rows.get(0).get(AspaceRefIdInfo.RECORD_ID_FIELD)); + assertEquals("03883_folder_5", rows.get(0).get(AspaceRefIdInfo.HOOK_ID_FIELD)); + assertEquals("8578708eda77e378b3a844a2166b815b", rows.get(0).get(AspaceRefIdInfo.REF_ID_FIELD)); + assertEquals("548", rows.get(1).get(AspaceRefIdInfo.RECORD_ID_FIELD)); + assertEquals("03883_folder_9", rows.get(1).get(AspaceRefIdInfo.HOOK_ID_FIELD)); + assertEquals("4c1196b46a06b21b1184fba0de1e84bd", rows.get(1).get(AspaceRefIdInfo.REF_ID_FIELD)); + assertEquals("549", rows.get(2).get(AspaceRefIdInfo.RECORD_ID_FIELD)); + assertEquals("03883_folder_9", rows.get(2).get(AspaceRefIdInfo.HOOK_ID_FIELD)); + assertEquals("4c1196b46a06b21b1184fba0de1e84bd", rows.get(2).get(AspaceRefIdInfo.REF_ID_FIELD)); + assertEquals(3, rows.size()); + } + } + + @Test + public void generateFromCdmIndexDbNoAspaceRefIdTest() throws Exception { + testHelper.indexExportData(Paths.get("src/test/resources/monograph_fields.csv"), "monograph"); + + Exception exception = assertThrows(InvalidProjectStateException.class, () -> { + service.generateAspaceRefIdMappingFromCdmIndexDb(); + }); + String expectedMessage = "Project has no contri field named hook id, " + + "and/or descri field named collection number, and/or refid field named aspace ref id"; + String actualMessage = exception.getMessage(); + + assertTrue(actualMessage.contains(expectedMessage)); + } + private CSVParser parser() throws IOException { Reader reader = Files.newBufferedReader(project.getAspaceRefIdMappingPath()); CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT diff --git a/src/test/resources/descriptions/03883/index/description/desc.all b/src/test/resources/descriptions/03883/index/description/desc.all index ea19a1c3..bceefabe 100644 --- a/src/test/resources/descriptions/03883/index/description/desc.all +++ b/src/test/resources/descriptions/03883/index/description/desc.all @@ -65,6 +65,7 @@ Volume1/03883_0005_0025.tif 231.jp2 +8578708eda77e378b3a844a2166b815b @@ -140,6 +141,7 @@ Volume1/03883_0009_0051.tif 711.jp2 +4c1196b46a06b21b1184fba0de1e84bd @@ -215,6 +217,7 @@ Volume1/03883_0009_0051.tif 711.jp2 +4c1196b46a06b21b1184fba0de1e84bd diff --git a/src/test/resources/findingaid_fields.csv b/src/test/resources/findingaid_fields.csv index 3109b390..69310681 100644 --- a/src/test/resources/findingaid_fields.csv +++ b/src/test/resources/findingaid_fields.csv @@ -59,4 +59,5 @@ dmcreated,dmcreated,Date created,false,y,n,y,n, dmmodified,dmmodified,Date modified,false,y,n,y,n, dmrecord,dmrecord,CONTENTdm number,false,y,n,y,n, find,find,CONTENTdm file name,false,y,n,y,n, -groupa,groupa,Group A,false,n,y,n,n,descri \ No newline at end of file +groupa,groupa,Group A,false,n,y,n,n,descri +refid,refid,Aspace Ref Id,false,n,y,n,n, \ No newline at end of file From 0f292884fe9713bf88fc749684e52ae4d24563f5 Mon Sep 17 00:00:00 2001 From: krwong Date: Wed, 1 Jul 2026 13:58:01 -0400 Subject: [PATCH 2/8] use ref_id instead of refid --- .../lib/boxc/migration/cdm/services/AspaceRefIdService.java | 4 ++-- src/test/resources/findingaid_fields.csv | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java index a678b9c8..010e9f9d 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java @@ -42,7 +42,7 @@ public class AspaceRefIdService { private Boolean projectHasContriAndDescri = null; private Boolean projectHasContriDescriAspaceRefId = null; - public static final String REF_ID_FIELD = "refid"; + public static final String REF_ID_FIELD = "ref_id"; public static final String[] HOOKID_REFID_CSV_HEADERS = {"cache_hookid", "normalized_cache_hookid", "collid", "ref_id", "ao_title", "tc_type", "tc_indicator", "sc_type", "sc_indicator", "gc_type", "gc_indicator", "aspace_hookid", "cdm_alias"}; @@ -275,7 +275,7 @@ private boolean hasProjectContriAndDescriFields() { private boolean hasProjectContriDescriAspaceRefIdFields() { if (projectHasContriDescriAspaceRefId == null) { // check if project has contri field named hook id, descri field named collection number, - // and refid field named aspace ref id + // and ref_id field named aspace ref id boolean hasContri = false; boolean hasDescri = false; boolean hasAspaceRefId = false; diff --git a/src/test/resources/findingaid_fields.csv b/src/test/resources/findingaid_fields.csv index 69310681..c35af454 100644 --- a/src/test/resources/findingaid_fields.csv +++ b/src/test/resources/findingaid_fields.csv @@ -60,4 +60,4 @@ dmmodified,dmmodified,Date modified,false,y,n,y,n, dmrecord,dmrecord,CONTENTdm number,false,y,n,y,n, find,find,CONTENTdm file name,false,y,n,y,n, groupa,groupa,Group A,false,n,y,n,n,descri -refid,refid,Aspace Ref Id,false,n,y,n,n, \ No newline at end of file +ref_id,ref_id,Aspace Ref Id,false,n,y,n,n, \ No newline at end of file From 7ae84586e5a4026437bd555596aaba72f50dfc7c Mon Sep 17 00:00:00 2001 From: krwong Date: Wed, 1 Jul 2026 14:36:29 -0400 Subject: [PATCH 3/8] fix test resource --- .../resources/descriptions/03883/index/description/desc.all | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/resources/descriptions/03883/index/description/desc.all b/src/test/resources/descriptions/03883/index/description/desc.all index bceefabe..159cece5 100644 --- a/src/test/resources/descriptions/03883/index/description/desc.all +++ b/src/test/resources/descriptions/03883/index/description/desc.all @@ -65,7 +65,7 @@ Volume1/03883_0005_0025.tif 231.jp2 -8578708eda77e378b3a844a2166b815b +8578708eda77e378b3a844a2166b815b @@ -141,7 +141,7 @@ Volume1/03883_0009_0051.tif 711.jp2 -4c1196b46a06b21b1184fba0de1e84bd +4c1196b46a06b21b1184fba0de1e84bd @@ -217,7 +217,7 @@ Volume1/03883_0009_0051.tif 711.jp2 -4c1196b46a06b21b1184fba0de1e84bd +4c1196b46a06b21b1184fba0de1e84bd From 80646e2ba97f891b63c7c23a13bdcfbb5bec871b Mon Sep 17 00:00:00 2001 From: krwong Date: Thu, 2 Jul 2026 13:55:52 -0400 Subject: [PATCH 4/8] wrap OR clauses in parens --- .../migration/cdm/services/AspaceRefIdService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java index 010e9f9d..a0427ba7 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java @@ -118,10 +118,10 @@ public void generateAspaceRefIdMappingFromCdmIndexDb() throws IOException { String query = "select " + CdmFieldInfo.CDM_ID + "," + FindingAidService.DESCRI_FIELD + "," + FindingAidService.CONTRI_FIELD + "," + REF_ID_FIELD + " from " + CdmIndexService.TB_NAME - + " where " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'" + + " where (" + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'" + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_COMPOUND_OBJECT + "'" + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_DOCUMENT_PDF + "'" - + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null" + + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null )" + " and " + CdmIndexService.PARENT_ID_FIELD + " is null" + " and " + FindingAidService.DESCRI_FIELD + " is not null" + " and " + FindingAidService.CONTRI_FIELD + " is not null"; @@ -164,10 +164,10 @@ private List getIds() { List ids = new ArrayList<>(); // for all work objects in the project (grouped works, compound objects, and single file works) String query = "select " + CdmFieldInfo.CDM_ID + " from " + CdmIndexService.TB_NAME - + " where " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'" + + " where (" + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'" + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_COMPOUND_OBJECT + "'" + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_DOCUMENT_PDF + "'" - + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null" + + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null )" + " and " + CdmIndexService.PARENT_ID_FIELD + " is null"; getIndexService(); @@ -193,10 +193,10 @@ private Map getIdsAndHookIds() { String query = "select " + CdmFieldInfo.CDM_ID + "," + FindingAidService.DESCRI_FIELD + "," + FindingAidService.CONTRI_FIELD + " from " + CdmIndexService.TB_NAME - + " where " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'" + + " where (" + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'" + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_COMPOUND_OBJECT + "'" + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_DOCUMENT_PDF + "'" - + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null" + + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null )" + " and " + CdmIndexService.PARENT_ID_FIELD + " is null" + " and " + FindingAidService.DESCRI_FIELD + " is not null" + " and " + FindingAidService.CONTRI_FIELD + " is not null"; From 56fc3516779368847dbdf13977be7e4bab8a181e Mon Sep 17 00:00:00 2001 From: krwong Date: Thu, 2 Jul 2026 15:06:21 -0400 Subject: [PATCH 5/8] ref_id desc also named ref_id --- .../lib/boxc/migration/cdm/services/AspaceRefIdService.java | 4 ++-- .../boxc/migration/cdm/services/AspaceRefIdServiceTest.java | 2 +- src/test/resources/findingaid_fields.csv | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java index a0427ba7..7aa57853 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java @@ -110,7 +110,7 @@ public void generateAspaceRefIdMappingFromCdmIndexDb() throws IOException { if (!hasProjectContriDescriAspaceRefIdFields()) { throw new InvalidProjectStateException("Project has no contri field named hook id, " + - "and/or descri field named collection number, and/or refid field named aspace ref id"); + "and/or descri field named collection number, and/or ref_id field named ref_id"); } try (BufferedWriter writer = Files.newBufferedWriter(getMappingPath()); @@ -296,7 +296,7 @@ private boolean hasProjectContriDescriAspaceRefIdFields() { && value.equalsIgnoreCase(FindingAidService.COLLECTION_NUMBER_FIELD_DESC)) { hasDescri = true; } else if (key.equalsIgnoreCase(REF_ID_FIELD) - && value.equalsIgnoreCase("aspace ref id")) { + && value.equalsIgnoreCase("ref_id")) { hasAspaceRefId = true; } } diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java index fd36b959..2eef3a81 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java @@ -223,7 +223,7 @@ public void generateFromCdmIndexDbNoAspaceRefIdTest() throws Exception { service.generateAspaceRefIdMappingFromCdmIndexDb(); }); String expectedMessage = "Project has no contri field named hook id, " + - "and/or descri field named collection number, and/or refid field named aspace ref id"; + "and/or descri field named collection number, and/or ref_id field named ref_id"; String actualMessage = exception.getMessage(); assertTrue(actualMessage.contains(expectedMessage)); diff --git a/src/test/resources/findingaid_fields.csv b/src/test/resources/findingaid_fields.csv index c35af454..47be87ad 100644 --- a/src/test/resources/findingaid_fields.csv +++ b/src/test/resources/findingaid_fields.csv @@ -60,4 +60,4 @@ dmmodified,dmmodified,Date modified,false,y,n,y,n, dmrecord,dmrecord,CONTENTdm number,false,y,n,y,n, find,find,CONTENTdm file name,false,y,n,y,n, groupa,groupa,Group A,false,n,y,n,n,descri -ref_id,ref_id,Aspace Ref Id,false,n,y,n,n, \ No newline at end of file +ref_id,ref_id,ref_id,false,n,y,n,n, \ No newline at end of file From a138c80e562b61315fd072f44688add734abff90 Mon Sep 17 00:00:00 2001 From: krwong Date: Thu, 2 Jul 2026 15:36:05 -0400 Subject: [PATCH 6/8] use hook_id instead of combining contri and descri --- .../cdm/services/AspaceRefIdService.java | 54 ++++++++----------- .../cdm/services/AspaceRefIdServiceTest.java | 3 +- .../03883/index/description/desc.all | 3 ++ src/test/resources/findingaid_fields.csv | 1 + 4 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java index 7aa57853..fe861928 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java @@ -40,8 +40,9 @@ public class AspaceRefIdService { private CdmIndexService indexService; private Path hookIdRefIdMapPath; private Boolean projectHasContriAndDescri = null; - private Boolean projectHasContriDescriAspaceRefId = null; + private Boolean projectHasHookIdRefId = null; + public static final String HOOK_ID_FIELD = "hook_id"; public static final String REF_ID_FIELD = "ref_id"; public static final String[] HOOKID_REFID_CSV_HEADERS = {"cache_hookid", "normalized_cache_hookid", "collid", "ref_id", "ao_title", "tc_type", "tc_indicator", "sc_type", "sc_indicator", "gc_type", "gc_indicator", @@ -108,15 +109,14 @@ public void generateAspaceRefIdMappingFromHookIdRefIdCsv() throws IOException { public void generateAspaceRefIdMappingFromCdmIndexDb() throws IOException { assertProjectStateValid(); - if (!hasProjectContriDescriAspaceRefIdFields()) { - throw new InvalidProjectStateException("Project has no contri field named hook id, " + - "and/or descri field named collection number, and/or ref_id field named ref_id"); + if (!hasProjectHookIdRefIdFields()) { + throw new InvalidProjectStateException("Project has no hook_id field named hook_id " + + "and/or ref_id field named ref_id"); } try (BufferedWriter writer = Files.newBufferedWriter(getMappingPath()); var csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT.withHeader(AspaceRefIdInfo.CSV_HEADERS))) { - String query = "select " + CdmFieldInfo.CDM_ID + "," + FindingAidService.DESCRI_FIELD + "," - + FindingAidService.CONTRI_FIELD + "," + REF_ID_FIELD + String query = "select " + CdmFieldInfo.CDM_ID + "," + HOOK_ID_FIELD + "," + REF_ID_FIELD + " from " + CdmIndexService.TB_NAME + " where (" + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'" + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_COMPOUND_OBJECT + "'" @@ -131,16 +131,13 @@ public void generateAspaceRefIdMappingFromCdmIndexDb() throws IOException { var stmt = conn.createStatement(); var rs = stmt.executeQuery(query); while (rs.next()) { - // if dmrecord, descri, contri, and ref_id fields are not blank, - // add dmrecord, descri_contri (hook id when combined), and ref_id to aspace ref id mapping + // if dmrecord, hook_id, and ref_id fields are not blank, + // add dmrecord, hook_id, and ref_id to aspace ref id mapping var dmrecord = rs.getString(1); - var descri = rs.getString(2); - var contri = rs.getString(3); - var refId = rs.getString(4); - if (!dmrecord.isBlank() && !descri.isBlank() && !contri.isBlank() & !refId.isBlank()) { - // remove -v from descri for matching purposes - csvPrinter.printRecord(dmrecord, - descri.replace("-z", "") + "_" + contri, refId); + var hookId = rs.getString(2); + var refId = rs.getString(3); + if (!dmrecord.isBlank() && !hookId.isBlank() & !refId.isBlank()) { + csvPrinter.printRecord(dmrecord, hookId, refId); } } } catch (SQLException e) { @@ -272,13 +269,11 @@ private boolean hasProjectContriAndDescriFields() { return projectHasContriAndDescri; } - private boolean hasProjectContriDescriAspaceRefIdFields() { - if (projectHasContriDescriAspaceRefId == null) { - // check if project has contri field named hook id, descri field named collection number, - // and ref_id field named aspace ref id - boolean hasContri = false; - boolean hasDescri = false; - boolean hasAspaceRefId = false; + private boolean hasProjectHookIdRefIdFields() { + if (projectHasHookIdRefId == null) { + // check if project has hook_id field named hook_id and ref_id field named ref_id + boolean hasHookId = false; + boolean hasRefId = false; fieldService.validateFieldsFile(project); CdmFieldInfo fieldInfo = fieldService.loadFieldsFromProject(project); Map fields = fieldInfo.getFields().stream() @@ -289,21 +284,18 @@ private boolean hasProjectContriDescriAspaceRefIdFields() { for (Map.Entry entry : fields.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); - if (key.equalsIgnoreCase(FindingAidService.CONTRI_FIELD) - && value.equalsIgnoreCase(FindingAidService.HOOK_ID_FIELD_DESC)) { - hasContri = true; - } else if (key.equalsIgnoreCase(FindingAidService.DESCRI_FIELD) - && value.equalsIgnoreCase(FindingAidService.COLLECTION_NUMBER_FIELD_DESC)) { - hasDescri = true; + if (key.equalsIgnoreCase(HOOK_ID_FIELD) + && value.equalsIgnoreCase(HOOK_ID_FIELD)) { + hasHookId = true; } else if (key.equalsIgnoreCase(REF_ID_FIELD) && value.equalsIgnoreCase("ref_id")) { - hasAspaceRefId = true; + hasRefId = true; } } - projectHasContriDescriAspaceRefId = hasContri && hasDescri && hasAspaceRefId; + projectHasHookIdRefId = hasHookId && hasRefId; } - return projectHasContriDescriAspaceRefId; + return projectHasHookIdRefId; } /** diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java index 2eef3a81..182800ec 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java @@ -222,8 +222,7 @@ public void generateFromCdmIndexDbNoAspaceRefIdTest() throws Exception { Exception exception = assertThrows(InvalidProjectStateException.class, () -> { service.generateAspaceRefIdMappingFromCdmIndexDb(); }); - String expectedMessage = "Project has no contri field named hook id, " + - "and/or descri field named collection number, and/or ref_id field named ref_id"; + String expectedMessage = "Project has no hook_id field named hook_id, and/or ref_id field named ref_id"; String actualMessage = exception.getMessage(); assertTrue(actualMessage.contains(expectedMessage)); diff --git a/src/test/resources/descriptions/03883/index/description/desc.all b/src/test/resources/descriptions/03883/index/description/desc.all index 159cece5..e452b4e6 100644 --- a/src/test/resources/descriptions/03883/index/description/desc.all +++ b/src/test/resources/descriptions/03883/index/description/desc.all @@ -65,6 +65,7 @@ Volume1/03883_0005_0025.tif 231.jp2 +03883_folder_5 8578708eda77e378b3a844a2166b815b @@ -141,6 +142,7 @@ Volume1/03883_0009_0051.tif 711.jp2 +03883_folder_9 4c1196b46a06b21b1184fba0de1e84bd @@ -217,6 +219,7 @@ Volume1/03883_0009_0051.tif 711.jp2 +03883_folder_9 4c1196b46a06b21b1184fba0de1e84bd diff --git a/src/test/resources/findingaid_fields.csv b/src/test/resources/findingaid_fields.csv index 47be87ad..618dce2a 100644 --- a/src/test/resources/findingaid_fields.csv +++ b/src/test/resources/findingaid_fields.csv @@ -60,4 +60,5 @@ dmmodified,dmmodified,Date modified,false,y,n,y,n, dmrecord,dmrecord,CONTENTdm number,false,y,n,y,n, find,find,CONTENTdm file name,false,y,n,y,n, groupa,groupa,Group A,false,n,y,n,n,descri +hook_id,hook_id,hook_id,false,n,y,n,y, ref_id,ref_id,ref_id,false,n,y,n,n, \ No newline at end of file From 4579542c718c72c49f71f793b28f8711652cadfa Mon Sep 17 00:00:00 2001 From: krwong Date: Thu, 2 Jul 2026 15:57:47 -0400 Subject: [PATCH 7/8] fix test --- .../lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java index 182800ec..cdfd31a6 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java @@ -222,7 +222,7 @@ public void generateFromCdmIndexDbNoAspaceRefIdTest() throws Exception { Exception exception = assertThrows(InvalidProjectStateException.class, () -> { service.generateAspaceRefIdMappingFromCdmIndexDb(); }); - String expectedMessage = "Project has no hook_id field named hook_id, and/or ref_id field named ref_id"; + String expectedMessage = "Project has no hook_id field named hook_id and/or ref_id field named ref_id"; String actualMessage = exception.getMessage(); assertTrue(actualMessage.contains(expectedMessage)); From f8edaee28457d52ef59b358e2c483b4e5f77bb49 Mon Sep 17 00:00:00 2001 From: krwong Date: Thu, 2 Jul 2026 16:45:14 -0400 Subject: [PATCH 8/8] fix query --- .../lib/boxc/migration/cdm/services/AspaceRefIdService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java index fe861928..12a18fc4 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java @@ -123,8 +123,8 @@ public void generateAspaceRefIdMappingFromCdmIndexDb() throws IOException { + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_DOCUMENT_PDF + "'" + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null )" + " and " + CdmIndexService.PARENT_ID_FIELD + " is null" - + " and " + FindingAidService.DESCRI_FIELD + " is not null" - + " and " + FindingAidService.CONTRI_FIELD + " is not null"; + + " and " + HOOK_ID_FIELD + " is not null" + + " and " + REF_ID_FIELD + " is not null"; getIndexService(); try (Connection conn = indexService.openDbConnection()) {