BXC-5682 generate ref id mappings from indexed ead to cdm metadata file

src/main/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommand.java

@@ -83,6 +83,30 @@ public int generateFromCsv() throws Exception {
         }
     }
 
+    @Command(name = "generate_from_index",
+            description = {"Generate the optional aspace ref id mapping file for this project " +
+                    "using the indexed ead to cdm metadata file, cdm_index.db.",
+                    "A ref_id_mapping.csv template will be created for this project, " +
+                            "with record ids, hook ids, and aspace ref ids populated."})
+    public int generateFromIndex() throws Exception {
+        long start = System.nanoTime();
+
+        try {
+            initialize();
+            aspaceRefIdService.generateAspaceRefIdMappingFromCdmIndexDb();
+            outputLogger.info("Aspace ref id mapping generated for {} in {}s", project.getProjectName(),
+                    (System.nanoTime() - start) / 1e9);
+            return 0;
+        } catch (MigrationException | IllegalArgumentException e) {
+            outputLogger.info("Cannot generate aspace ref id mapping: {}", e.getMessage());
+            return 1;
+        } catch (Exception e) {
+            log.error("Failed to generate aspace ref id template", e);
+            outputLogger.info("Failed to generate aspace ref id template: {}", e.getMessage(), e);
+            return 1;
+        }
+    }
+
     @Command(name = "validate",
             description = {"Validate the aspace ref id mappings for this project"})
     public int validate(@Option(names = { "-f", "--force"},

src/main/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdService.java

@@ -40,7 +40,10 @@ public class AspaceRefIdService {
     private CdmIndexService indexService;
     private Path hookIdRefIdMapPath;
     private Boolean projectHasContriAndDescri = null;
+    private Boolean projectHasHookIdRefId = null;
 
+    public static final String HOOK_ID_FIELD = "hook_id";
+    public static final String REF_ID_FIELD = "ref_id";
     public static final String[] HOOKID_REFID_CSV_HEADERS = {"cache_hookid", "normalized_cache_hookid", "collid",
             "ref_id", "ao_title", "tc_type", "tc_indicator", "sc_type", "sc_indicator", "gc_type", "gc_indicator",
             "aspace_hookid", "cdm_alias"};
@@ -98,6 +101,53 @@ public void generateAspaceRefIdMappingFromHookIdRefIdCsv() throws IOException {
         setUpdatedDate(Instant.now());
     }
 
+    /**
+     * Generate the aspace ref id mapping using the indexed ead to cdm metadata file (cdm_index.db)
+     * Record ids, hook ids, and ref ids populated
+     * @throws Exception
+     */
+    public void generateAspaceRefIdMappingFromCdmIndexDb() throws IOException {
+        assertProjectStateValid();
+
+        if (!hasProjectHookIdRefIdFields()) {
+            throw new InvalidProjectStateException("Project has no hook_id field named hook_id " +
+                    "and/or ref_id field named ref_id");
+        }
+
+        try (BufferedWriter writer = Files.newBufferedWriter(getMappingPath());
+             var csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT.withHeader(AspaceRefIdInfo.CSV_HEADERS))) {
+            String query = "select " + CdmFieldInfo.CDM_ID + "," + HOOK_ID_FIELD + "," + REF_ID_FIELD
+                    + " from " + CdmIndexService.TB_NAME
+                    + " where (" + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'"
+                    + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_COMPOUND_OBJECT + "'"
+                    + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_DOCUMENT_PDF + "'"
+                    + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null )"
+                    + " and " + CdmIndexService.PARENT_ID_FIELD + " is null"
+                    + " and " + HOOK_ID_FIELD + " is not null"
+                    + " and " + REF_ID_FIELD + " is not null";
+
+            getIndexService();
+            try (Connection conn = indexService.openDbConnection()) {
+                var stmt = conn.createStatement();
+                var rs = stmt.executeQuery(query);
+                while (rs.next()) {
+                    // if dmrecord, hook_id, and ref_id fields are not blank,
+                    // add dmrecord, hook_id, and ref_id to aspace ref id mapping
+                    var dmrecord = rs.getString(1);
+                    var hookId = rs.getString(2);
+                    var refId = rs.getString(3);
+                    if (!dmrecord.isBlank() && !hookId.isBlank() & !refId.isBlank()) {
+                        csvPrinter.printRecord(dmrecord, hookId, refId);
+                    }
+                }
+            } catch (SQLException e) {
+                throw new MigrationException("Error interacting with export index", e);
+            }
+        }
+
+        setUpdatedDate(Instant.now());
+    }
+
     protected void setUpdatedDate(Instant timestamp) throws IOException {
         project.getProjectProperties().setAspaceRefIdMappingsUpdatedDate(timestamp);
         ProjectPropertiesSerialization.write(project);
@@ -111,10 +161,10 @@ private List<String> getIds() {
         List<String> ids = new ArrayList<>();
         // for all work objects in the project (grouped works, compound objects, and single file works)
         String query = "select " + CdmFieldInfo.CDM_ID + " from " + CdmIndexService.TB_NAME
-                + " where " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'"
+                + " where (" + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'"
                 + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_COMPOUND_OBJECT + "'"
                 + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_DOCUMENT_PDF + "'"
-                + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null"
+                + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null )"
                 + " and " + CdmIndexService.PARENT_ID_FIELD + " is null";
 
         getIndexService();
@@ -140,10 +190,10 @@ private Map<String,String> getIdsAndHookIds() {
         String query = "select " + CdmFieldInfo.CDM_ID + "," + FindingAidService.DESCRI_FIELD + ","
                 + FindingAidService.CONTRI_FIELD
                 + " from " + CdmIndexService.TB_NAME
-                + " where " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'"
+                + " where (" + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_GROUPED_WORK + "'"
                 + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_COMPOUND_OBJECT + "'"
                 + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " = '" + CdmIndexService.ENTRY_TYPE_DOCUMENT_PDF + "'"
-                + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null"
+                + " or " + CdmIndexService.ENTRY_TYPE_FIELD + " is null )"
                 + " and " + CdmIndexService.PARENT_ID_FIELD + " is null"
                 + " and " + FindingAidService.DESCRI_FIELD + " is not null"
                 + " and " + FindingAidService.CONTRI_FIELD + " is not null";
@@ -219,6 +269,35 @@ private boolean hasProjectContriAndDescriFields() {
         return projectHasContriAndDescri;
     }
 
+    private boolean hasProjectHookIdRefIdFields() {
+        if (projectHasHookIdRefId == null) {
+            // check if project has hook_id field named hook_id and ref_id field named ref_id
+            boolean hasHookId = false;
+            boolean hasRefId = false;
+            fieldService.validateFieldsFile(project);
+            CdmFieldInfo fieldInfo = fieldService.loadFieldsFromProject(project);
+            Map<String, String> fields = fieldInfo.getFields().stream()
+                    .filter(f -> !f.getSkipExport())
+                    .collect(Collectors.toMap(CdmFieldInfo.CdmFieldEntry::getNickName,
+                            CdmFieldInfo.CdmFieldEntry::getDescription));
+
+            for (Map.Entry<String, String> entry : fields.entrySet()) {
+                String key = entry.getKey();
+                String value = entry.getValue();
+                if (key.equalsIgnoreCase(HOOK_ID_FIELD)
+                        && value.equalsIgnoreCase(HOOK_ID_FIELD)) {
+                    hasHookId = true;
+                } else if (key.equalsIgnoreCase(REF_ID_FIELD)
+                    && value.equalsIgnoreCase("ref_id")) {
+                    hasRefId = true;
+                }
+            }
+
+            projectHasHookIdRefId = hasHookId && hasRefId;
+        }
+        return projectHasHookIdRefId;
+    }
+
     /**
      * @return the aspace ref id mapping info for the configured project
      * @throws IOException

src/test/java/edu/unc/lib/boxc/migration/cdm/AspaceRefIdCommandIT.java

@@ -97,6 +97,21 @@ public void generateFromCsvAspaceRefIdMappingSucceedsTest() throws Exception {
         assertUpdatedDatePresent();
     }
 
+    @Test
+    public void generateFromIndexAspaceRefIdMappingSucceedsTest() throws Exception {
+        indexExportSamples();
+
+        String[] args = new String[] {
+                "-w", project.getProjectPath().toString(),
+                "--env-config", chompbConfigPath,
+                "aspace_ref_id", "generate_from_index"};
+        executeExpectSuccess(args);
+
+        assertTrue(Files.exists(project.getAspaceRefIdMappingPath()));
+
+        assertUpdatedDatePresent();
+    }
+
     @Test
     public void validateValidTest() throws Exception {
         indexExportSamples();

src/test/java/edu/unc/lib/boxc/migration/cdm/services/AspaceRefIdServiceTest.java

@@ -194,6 +194,40 @@ public void generateFromHookIdRefIdCsvNoContriDescriTest() throws Exception {
         assertTrue(actualMessage.contains(expectedMessage));
     }
 
+    @Test
+    public void generateFromCdmIndexDbTest() throws Exception {
+        testHelper.indexExportData(Paths.get("src/test/resources/findingaid_fields.csv"), "03883");
+        service.generateAspaceRefIdMappingFromCdmIndexDb();
+
+        assertTrue(Files.exists(project.getAspaceRefIdMappingPath()));
+        try (CSVParser csvParser = parser()) {
+            List<CSVRecord> rows = csvParser.getRecords();
+            assertEquals("0", rows.get(0).get(AspaceRefIdInfo.RECORD_ID_FIELD));
+            assertEquals("03883_folder_5", rows.get(0).get(AspaceRefIdInfo.HOOK_ID_FIELD));
+            assertEquals("8578708eda77e378b3a844a2166b815b", rows.get(0).get(AspaceRefIdInfo.REF_ID_FIELD));
+            assertEquals("548", rows.get(1).get(AspaceRefIdInfo.RECORD_ID_FIELD));
+            assertEquals("03883_folder_9", rows.get(1).get(AspaceRefIdInfo.HOOK_ID_FIELD));
+            assertEquals("4c1196b46a06b21b1184fba0de1e84bd", rows.get(1).get(AspaceRefIdInfo.REF_ID_FIELD));
+            assertEquals("549", rows.get(2).get(AspaceRefIdInfo.RECORD_ID_FIELD));
+            assertEquals("03883_folder_9", rows.get(2).get(AspaceRefIdInfo.HOOK_ID_FIELD));
+            assertEquals("4c1196b46a06b21b1184fba0de1e84bd", rows.get(2).get(AspaceRefIdInfo.REF_ID_FIELD));
+            assertEquals(3, rows.size());
+        }
+    }
+
+    @Test
+    public void generateFromCdmIndexDbNoAspaceRefIdTest() throws Exception {
+        testHelper.indexExportData(Paths.get("src/test/resources/monograph_fields.csv"), "monograph");
+
+        Exception exception = assertThrows(InvalidProjectStateException.class, () -> {
+            service.generateAspaceRefIdMappingFromCdmIndexDb();
+        });
+        String expectedMessage = "Project has no hook_id field named hook_id and/or ref_id field named ref_id";
+        String actualMessage = exception.getMessage();
+
+        assertTrue(actualMessage.contains(expectedMessage));
+    }
+
     private CSVParser parser() throws IOException {
         Reader reader = Files.newBufferedReader(project.getAspaceRefIdMappingPath());
         CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT

src/test/resources/descriptions/03883/index/description/desc.all

@@ -65,6 +65,8 @@
 <duracl></duracl>
 <fullrs>Volume1/03883_0005_0025.tif</fullrs>
 <find>231.jp2</find>
+<hook_id>03883_folder_5</hook_id>
+<ref_id>8578708eda77e378b3a844a2166b815b</ref_id>
 <dmaccess></dmaccess>
 <dmimage></dmimage>
 <dmad1></dmad1>
@@ -140,6 +142,8 @@
 <duracl></duracl>
 <fullrs>Volume1/03883_0009_0051.tif</fullrs>
 <find>711.jp2</find>
+<hook_id>03883_folder_9</hook_id>
+<ref_id>4c1196b46a06b21b1184fba0de1e84bd</ref_id>
 <dmaccess></dmaccess>
 <dmimage></dmimage>
 <dmad1></dmad1>
@@ -215,6 +219,8 @@
 <duracl></duracl>
 <fullrs>Volume1/03883_0009_0051.tif</fullrs>
 <find>711.jp2</find>
+<hook_id>03883_folder_9</hook_id>
+<ref_id>4c1196b46a06b21b1184fba0de1e84bd</ref_id>
 <dmaccess></dmaccess>
 <dmimage></dmimage>
 <dmad1></dmad1>

src/test/resources/findingaid_fields.csv

@@ -59,4 +59,6 @@ dmcreated,dmcreated,Date created,false,y,n,y,n,
 dmmodified,dmmodified,Date modified,false,y,n,y,n,
 dmrecord,dmrecord,CONTENTdm number,false,y,n,y,n,
 find,find,CONTENTdm file name,false,y,n,y,n,
-groupa,groupa,Group A,false,n,y,n,n,descri
+groupa,groupa,Group A,false,n,y,n,n,descri
+hook_id,hook_id,hook_id,false,n,y,n,y,
+ref_id,ref_id,ref_id,false,n,y,n,n,