diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommand.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommand.java index 9861c197..70f1325e 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommand.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommand.java @@ -83,6 +83,30 @@ public int generateFromCsv() throws Exception { } } + @Command(name = "generate_from_index", + description = {"Generate the optional aspace ref id mapping file for this project " + + "using the indexed ead to cdm metadata file, cdm_index.db.", + "A ref_id_mapping.csv template will be created for this project, " + + "with record ids, hook ids, and aspace ref ids populated."}) + public int generateFromIndex() throws Exception { + long start = System.nanoTime(); + + try { + initialize(); + aspaceRefIdService.generateAspaceRefIdMappingFromCdmIndexDb(); + outputLogger.info("Aspace ref id mapping generated for {} in {}s", project.getProjectName(), + (System.nanoTime() - start) / 1e9); + return 0; + } catch (MigrationException | IllegalArgumentException e) { + outputLogger.info("Cannot generate aspace ref id mapping: {}", e.getMessage()); + return 1; + } catch (Exception e) { + log.error("Failed to generate aspace ref id template", e); + outputLogger.info("Failed to generate aspace ref id template: {}", e.getMessage(), e); + return 1; + } + } + @Command(name = "validate", description = {"Validate the aspace ref id mappings for this project"}) public int validate(@Option(names = { "-f", "--force"}, diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java index 0d9f9c5e..12a18fc4 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java @@ -40,7 +40,10 @@ public class AspaceRefIdService { private CdmIndexService indexService; private Path hookIdRefIdMapPath; private Boolean projectHasContriAndDescri = null; + private Boolean projectHasHookIdRefId = null; + public static final String HOOK_ID_FIELD = "hook_id"; + public static final String REF_ID_FIELD = "ref_id"; public static final String[] HOOKID_REFID_CSV_HEADERS = {"cache_hookid", "normalized_cache_hookid", "collid", "ref_id", "ao_title", "tc_type", "tc_indicator", "sc_type", "sc_indicator", "gc_type", "gc_indicator", "aspace_hookid", "cdm_alias"}; @@ -98,6 +101,53 @@ public void generateAspaceRefIdMappingFromHookIdRefIdCsv() throws IOException { setUpdatedDate(Instant.now()); } + /** + * Generate the aspace ref id mapping using the indexed ead to cdm metadata file (cdm_index.db) + * Record ids, hook ids, and ref ids populated + * @throws Exception + */ + public void generateAspaceRefIdMappingFromCdmIndexDb() throws IOException { + assertProjectStateValid(); + + if (!hasProjectHookIdRefIdFields()) { + throw new InvalidProjectStateException("Project has no hook_id field named hook_id " + + "and/or ref_id field named ref_id"); + } + + try (BufferedWriter writer = Files.newBufferedWriter(getMappingPath()); + var csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT.withHeader(AspaceRefIdInfo.CSV_HEADERS))) { + String query = "select " + CdmFieldInfo.CDM_ID + "," + HOOK_ID_FIELD + "," + REF_ID_FIELD + + " from " + CdmIndexService.TB_NAME + + " where (" + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'" + + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_COMPOUND_OBJECT + "'" + + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_DOCUMENT_PDF + "'" + + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null )" + + " and " + CdmIndexService.PARENT_ID_FIELD + " is null" + + " and " + HOOK_ID_FIELD + " is not null" + + " and " + REF_ID_FIELD + " is not null"; + + getIndexService(); + try (Connection conn = indexService.openDbConnection()) { + var stmt = conn.createStatement(); + var rs = stmt.executeQuery(query); + while (rs.next()) { + // if dmrecord, hook_id, and ref_id fields are not blank, + // add dmrecord, hook_id, and ref_id to aspace ref id mapping + var dmrecord = rs.getString(1); + var hookId = rs.getString(2); + var refId = rs.getString(3); + if (!dmrecord.isBlank() && !hookId.isBlank() & !refId.isBlank()) { + csvPrinter.printRecord(dmrecord, hookId, refId); + } + } + } catch (SQLException e) { + throw new MigrationException("Error interacting with export index", e); + } + } + + setUpdatedDate(Instant.now()); + } + protected void setUpdatedDate(Instant timestamp) throws IOException { project.getProjectProperties().setAspaceRefIdMappingsUpdatedDate(timestamp); ProjectPropertiesSerialization.write(project); @@ -111,10 +161,10 @@ private List getIds() { List ids = new ArrayList<>(); // for all work objects in the project (grouped works, compound objects, and single file works) String query = "select " + CdmFieldInfo.CDM_ID + " from " + CdmIndexService.TB_NAME - + " where " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'" + + " where (" + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'" + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_COMPOUND_OBJECT + "'" + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_DOCUMENT_PDF + "'" - + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null" + + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null )" + " and " + CdmIndexService.PARENT_ID_FIELD + " is null"; getIndexService(); @@ -140,10 +190,10 @@ private Map getIdsAndHookIds() { String query = "select " + CdmFieldInfo.CDM_ID + "," + FindingAidService.DESCRI_FIELD + "," + FindingAidService.CONTRI_FIELD + " from " + CdmIndexService.TB_NAME - + " where " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'" + + " where (" + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'" + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_COMPOUND_OBJECT + "'" + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_DOCUMENT_PDF + "'" - + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null" + + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null )" + " and " + CdmIndexService.PARENT_ID_FIELD + " is null" + " and " + FindingAidService.DESCRI_FIELD + " is not null" + " and " + FindingAidService.CONTRI_FIELD + " is not null"; @@ -219,6 +269,35 @@ private boolean hasProjectContriAndDescriFields() { return projectHasContriAndDescri; } + private boolean hasProjectHookIdRefIdFields() { + if (projectHasHookIdRefId == null) { + // check if project has hook_id field named hook_id and ref_id field named ref_id + boolean hasHookId = false; + boolean hasRefId = false; + fieldService.validateFieldsFile(project); + CdmFieldInfo fieldInfo = fieldService.loadFieldsFromProject(project); + Map fields = fieldInfo.getFields().stream() + .filter(f -> !f.getSkipExport()) + .collect(Collectors.toMap(CdmFieldInfo.CdmFieldEntry::getNickName, + CdmFieldInfo.CdmFieldEntry::getDescription)); + + for (Map.Entry entry : fields.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (key.equalsIgnoreCase(HOOK_ID_FIELD) + && value.equalsIgnoreCase(HOOK_ID_FIELD)) { + hasHookId = true; + } else if (key.equalsIgnoreCase(REF_ID_FIELD) + && value.equalsIgnoreCase("ref_id")) { + hasRefId = true; + } + } + + projectHasHookIdRefId = hasHookId && hasRefId; + } + return projectHasHookIdRefId; + } + /** * @return the aspace ref id mapping info for the configured project * @throws IOException diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommandIT.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommandIT.java index bd316d17..d5b6a824 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommandIT.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommandIT.java @@ -97,6 +97,21 @@ public void generateFromCsvAspaceRefIdMappingSucceedsTest() throws Exception { assertUpdatedDatePresent(); } + @Test + public void generateFromIndexAspaceRefIdMappingSucceedsTest() throws Exception { + indexExportSamples(); + + String[] args = new String[] { + "-w", project.getProjectPath().toString(), + "--env-config", chompbConfigPath, + "aspace_ref_id", "generate_from_index"}; + executeExpectSuccess(args); + + assertTrue(Files.exists(project.getAspaceRefIdMappingPath())); + + assertUpdatedDatePresent(); + } + @Test public void validateValidTest() throws Exception { indexExportSamples(); diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java index 8b1c993d..cdfd31a6 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java @@ -194,6 +194,40 @@ public void generateFromHookIdRefIdCsvNoContriDescriTest() throws Exception { assertTrue(actualMessage.contains(expectedMessage)); } + @Test + public void generateFromCdmIndexDbTest() throws Exception { + testHelper.indexExportData(Paths.get("src/test/resources/findingaid_fields.csv"), "03883"); + service.generateAspaceRefIdMappingFromCdmIndexDb(); + + assertTrue(Files.exists(project.getAspaceRefIdMappingPath())); + try (CSVParser csvParser = parser()) { + List rows = csvParser.getRecords(); + assertEquals("0", rows.get(0).get(AspaceRefIdInfo.RECORD_ID_FIELD)); + assertEquals("03883_folder_5", rows.get(0).get(AspaceRefIdInfo.HOOK_ID_FIELD)); + assertEquals("8578708eda77e378b3a844a2166b815b", rows.get(0).get(AspaceRefIdInfo.REF_ID_FIELD)); + assertEquals("548", rows.get(1).get(AspaceRefIdInfo.RECORD_ID_FIELD)); + assertEquals("03883_folder_9", rows.get(1).get(AspaceRefIdInfo.HOOK_ID_FIELD)); + assertEquals("4c1196b46a06b21b1184fba0de1e84bd", rows.get(1).get(AspaceRefIdInfo.REF_ID_FIELD)); + assertEquals("549", rows.get(2).get(AspaceRefIdInfo.RECORD_ID_FIELD)); + assertEquals("03883_folder_9", rows.get(2).get(AspaceRefIdInfo.HOOK_ID_FIELD)); + assertEquals("4c1196b46a06b21b1184fba0de1e84bd", rows.get(2).get(AspaceRefIdInfo.REF_ID_FIELD)); + assertEquals(3, rows.size()); + } + } + + @Test + public void generateFromCdmIndexDbNoAspaceRefIdTest() throws Exception { + testHelper.indexExportData(Paths.get("src/test/resources/monograph_fields.csv"), "monograph"); + + Exception exception = assertThrows(InvalidProjectStateException.class, () -> { + service.generateAspaceRefIdMappingFromCdmIndexDb(); + }); + String expectedMessage = "Project has no hook_id field named hook_id and/or ref_id field named ref_id"; + String actualMessage = exception.getMessage(); + + assertTrue(actualMessage.contains(expectedMessage)); + } + private CSVParser parser() throws IOException { Reader reader = Files.newBufferedReader(project.getAspaceRefIdMappingPath()); CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT diff --git a/src/test/resources/descriptions/03883/index/description/desc.all b/src/test/resources/descriptions/03883/index/description/desc.all index ea19a1c3..e452b4e6 100644 --- a/src/test/resources/descriptions/03883/index/description/desc.all +++ b/src/test/resources/descriptions/03883/index/description/desc.all @@ -65,6 +65,8 @@ Volume1/03883_0005_0025.tif 231.jp2 +03883_folder_5 +8578708eda77e378b3a844a2166b815b @@ -140,6 +142,8 @@ Volume1/03883_0009_0051.tif 711.jp2 +03883_folder_9 +4c1196b46a06b21b1184fba0de1e84bd @@ -215,6 +219,8 @@ Volume1/03883_0009_0051.tif 711.jp2 +03883_folder_9 +4c1196b46a06b21b1184fba0de1e84bd diff --git a/src/test/resources/findingaid_fields.csv b/src/test/resources/findingaid_fields.csv index 3109b390..618dce2a 100644 --- a/src/test/resources/findingaid_fields.csv +++ b/src/test/resources/findingaid_fields.csv @@ -59,4 +59,6 @@ dmcreated,dmcreated,Date created,false,y,n,y,n, dmmodified,dmmodified,Date modified,false,y,n,y,n, dmrecord,dmrecord,CONTENTdm number,false,y,n,y,n, find,find,CONTENTdm file name,false,y,n,y,n, -groupa,groupa,Group A,false,n,y,n,n,descri \ No newline at end of file +groupa,groupa,Group A,false,n,y,n,n,descri +hook_id,hook_id,hook_id,false,n,y,n,y, +ref_id,ref_id,ref_id,false,n,y,n,n, \ No newline at end of file