From 798240163e42ef5df85eeca52a69029a738fe057 Mon Sep 17 00:00:00 2001 From: krwong Date: Wed, 30 Sep 2026 17:06:16 -0400 Subject: [PATCH] remove StreamingMetadataService and duracloud streaming rdf fields from chompb --- .../migration/cdm/AccessFilesCommand.java | 7 - .../migration/cdm/AggregateFilesCommand.java | 7 - .../lib/boxc/migration/cdm/SipsCommand.java | 7 - .../migration/cdm/SourceFilesCommand.java | 9 -- .../lib/boxc/migration/cdm/StatusCommand.java | 7 - .../migration/cdm/services/SipService.java | 6 - .../cdm/services/SourceFileService.java | 11 +- .../services/StreamingMetadataService.java | 134 ------------------ .../cdm/services/sips/WorkGenerator.java | 53 ++----- .../services/sips/WorkGeneratorFactory.java | 7 - .../cdm/status/ProjectStatusService.java | 8 -- .../cdm/status/SourceFilesStatusService.java | 8 -- .../cdm/validators/SourceFilesValidator.java | 8 +- .../migration/cdm/CompleteMigrationIT.java | 92 ------------ .../lib/boxc/migration/cdm/SipsCommandIT.java | 55 ------- .../migration/cdm/SourceFilesCommandIT.java | 37 ----- .../boxc/migration/cdm/StatusCommandIT.java | 3 +- .../FieldAssessmentTemplateServiceTest.java | 2 +- .../cdm/services/SipServiceTest.java | 55 ------- .../cdm/services/SourceFileServiceTest.java | 19 +-- .../StreamingMetadataServiceTest.java | 85 ----------- .../status/SourceFilesStatusServiceTest.java | 1 - .../migration/cdm/test/SipServiceHelper.java | 16 --- .../validators/SourceFilesValidatorTest.java | 22 --- src/test/resources/cdm_fields_resp.json | 30 ---- .../03883-z/index/description/desc.all | 6 - .../03883/index/description/desc.all | 6 - .../index/description/desc.all | 4 - src/test/resources/gilmer_fields.csv | 2 - src/test/resources/pdf_fields.csv | 2 - src/test/resources/roy_brown/cdm_fields.csv | 2 - 31 files changed, 16 insertions(+), 695 deletions(-) delete mode 100644 src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java delete mode 100644 src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/AccessFilesCommand.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/AccessFilesCommand.java index ecbc5fdc..c3e72912 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/AccessFilesCommand.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/AccessFilesCommand.java @@ -9,7 +9,6 @@ import edu.unc.lib.boxc.migration.cdm.options.GenerateSourceFileMappingOptions; import edu.unc.lib.boxc.migration.cdm.services.CdmFieldService; -import edu.unc.lib.boxc.migration.cdm.services.StreamingMetadataService; import edu.unc.lib.boxc.migration.cdm.status.SourceFilesSummaryService; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -39,7 +38,6 @@ public class AccessFilesCommand { private CLIMain parentCommand; private MigrationProject project; - private StreamingMetadataService streamingMetadataService; private SourceFilesSummaryService summaryService; private AccessFileService accessService; @@ -137,13 +135,8 @@ private void initialize(boolean dryRun) throws IOException { CdmFieldService fieldService = new CdmFieldService(); CdmIndexService indexService = new CdmIndexService(); indexService.setProject(project); - streamingMetadataService = new StreamingMetadataService(); - streamingMetadataService.setProject(project); - streamingMetadataService.setFieldService(fieldService); - streamingMetadataService.setIndexService(indexService); accessService = new AccessFileService(); accessService.setIndexService(indexService); - accessService.setStreamingMetadataService(streamingMetadataService); accessService.setProject(project); summaryService = new SourceFilesSummaryService(); summaryService.setProject(project); diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/AggregateFilesCommand.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/AggregateFilesCommand.java index b43103e0..3bb19697 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/AggregateFilesCommand.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/AggregateFilesCommand.java @@ -8,7 +8,6 @@ import edu.unc.lib.boxc.migration.cdm.services.CdmFieldService; import edu.unc.lib.boxc.migration.cdm.services.CdmIndexService; import edu.unc.lib.boxc.migration.cdm.services.MigrationProjectFactory; -import edu.unc.lib.boxc.migration.cdm.services.StreamingMetadataService; import edu.unc.lib.boxc.migration.cdm.status.SourceFilesSummaryService; import edu.unc.lib.boxc.migration.cdm.validators.AggregateFilesValidator; import org.apache.commons.lang3.StringUtils; @@ -40,7 +39,6 @@ public class AggregateFilesCommand { private CdmFieldService fieldService; private CdmIndexService indexService; private SourceFilesSummaryService summaryService; - private StreamingMetadataService streamingMetadataService; @CommandLine.Command(name = "generate", description = { @@ -139,14 +137,9 @@ private void initialize(boolean sortBottom, boolean dryRun) throws IOException { fieldService = new CdmFieldService(); indexService = new CdmIndexService(); indexService.setProject(project); - streamingMetadataService = new StreamingMetadataService(); - streamingMetadataService.setProject(project); - streamingMetadataService.setFieldService(fieldService); - streamingMetadataService.setIndexService(indexService); aggregateService = new AggregateFileMappingService(sortBottom); aggregateService.setIndexService(indexService); aggregateService.setProject(project); - aggregateService.setStreamingMetadataService(streamingMetadataService); summaryService = new SourceFilesSummaryService(); summaryService.setProject(project); summaryService.setDryRun(dryRun); diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/SipsCommand.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/SipsCommand.java index 14c2f4b9..0075e756 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/SipsCommand.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/SipsCommand.java @@ -11,7 +11,6 @@ import edu.unc.lib.boxc.migration.cdm.services.AltTextService; import edu.unc.lib.boxc.migration.cdm.services.AspaceRefIdService; import edu.unc.lib.boxc.migration.cdm.services.CdmFieldService; -import edu.unc.lib.boxc.migration.cdm.services.StreamingMetadataService; import org.slf4j.Logger; import edu.unc.lib.boxc.migration.cdm.exceptions.MigrationException; @@ -53,7 +52,6 @@ public class SipsCommand { private AggregateFileMappingService aggregateTopMappingService; private AggregateFileMappingService aggregateBottomMappingService; private CdmFieldService fieldService; - private StreamingMetadataService streamingMetadataService; private PIDMinter pidMinter; private PremisLoggerFactoryImpl premisLoggerFactory; private SipService sipService; @@ -155,10 +153,6 @@ private void initialize() throws IOException { aggregateBottomMappingService.setIndexService(indexService); aggregateBottomMappingService.setProject(project); fieldService = new CdmFieldService(); - streamingMetadataService = new StreamingMetadataService(); - streamingMetadataService.setProject(project); - streamingMetadataService.setFieldService(fieldService); - streamingMetadataService.setIndexService(indexService); sipService = new SipService(); sipService.setIndexService(indexService); @@ -173,6 +167,5 @@ private void initialize() throws IOException { sipService.setChompbConfig(parentCommand.getChompbConfig()); sipService.setAggregateTopMappingService(aggregateTopMappingService); sipService.setAggregateBottomMappingService(aggregateBottomMappingService); - sipService.setStreamingMetadataService(streamingMetadataService); } } diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/SourceFilesCommand.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/SourceFilesCommand.java index 8c0182bc..95304189 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/SourceFilesCommand.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/SourceFilesCommand.java @@ -15,7 +15,6 @@ import edu.unc.lib.boxc.migration.cdm.services.CdmExportFilesService; import edu.unc.lib.boxc.migration.cdm.services.CdmFieldService; import edu.unc.lib.boxc.migration.cdm.services.CdmFileRetrievalService; -import edu.unc.lib.boxc.migration.cdm.services.StreamingMetadataService; import edu.unc.lib.boxc.migration.cdm.status.SourceFilesSummaryService; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -50,7 +49,6 @@ public class SourceFilesCommand { private CdmExportFilesService exportFilesService; private CdmFieldService fieldService; private SourceFilesSummaryService summaryService; - private StreamingMetadataService streamingMetadataService; @Command(name = "generate", description = { @@ -95,7 +93,6 @@ public int validate(@Option(names = { "-f", "--force"}, initialize(false); SourceFilesValidator validator = new SourceFilesValidator(); validator.setProject(project); - validator.setStreamingMetadataService(streamingMetadataService); List errors = validator.validateMappings(force); if (errors.isEmpty()) { outputLogger.info("PASS: Source file mapping at path {} is valid", @@ -127,7 +124,6 @@ public int status() throws Exception { initialize(false); SourceFilesStatusService statusService = new SourceFilesStatusService(); statusService.setProject(project); - statusService.setStreamingMetadataService(streamingMetadataService); statusService.report(parentCommand.getVerbosity()); return 0; @@ -205,13 +201,8 @@ private void initialize(boolean dryRun) throws IOException { fieldService = new CdmFieldService(); indexService = new CdmIndexService(); indexService.setProject(project); - streamingMetadataService = new StreamingMetadataService(); - streamingMetadataService.setProject(project); - streamingMetadataService.setFieldService(fieldService); - streamingMetadataService.setIndexService(indexService); sourceService = new SourceFileService(); sourceService.setIndexService(indexService); - sourceService.setStreamingMetadataService(streamingMetadataService); sourceService.setProject(project); summaryService = new SourceFilesSummaryService(); summaryService.setProject(project); diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/StatusCommand.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/StatusCommand.java index 44e01183..98ea3ff5 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/StatusCommand.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/StatusCommand.java @@ -12,7 +12,6 @@ import edu.unc.lib.boxc.migration.cdm.services.CdmFieldService; import edu.unc.lib.boxc.migration.cdm.services.CdmIndexService; import edu.unc.lib.boxc.migration.cdm.services.MigrationProjectFactory; -import edu.unc.lib.boxc.migration.cdm.services.StreamingMetadataService; import edu.unc.lib.boxc.migration.cdm.status.ProjectStatusService; import org.slf4j.Logger; import picocli.CommandLine.Command; @@ -31,7 +30,6 @@ public class StatusCommand implements Callable { private ProjectStatusService statusService; private CdmFieldService fieldService; private CdmIndexService indexService; - private StreamingMetadataService streamingMetadataService; private MigrationProject project; @Override @@ -56,12 +54,7 @@ private void initialize() throws IOException { fieldService = new CdmFieldService(); indexService = new CdmIndexService(); indexService.setProject(project); - streamingMetadataService = new StreamingMetadataService(); - streamingMetadataService.setProject(project); - streamingMetadataService.setFieldService(fieldService); - streamingMetadataService.setIndexService(indexService); statusService = new ProjectStatusService(); statusService.setProject(project); - statusService.setStreamingMetadataService(streamingMetadataService); } } diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SipService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SipService.java index ae4318ee..679b661a 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SipService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SipService.java @@ -76,7 +76,6 @@ public class SipService { private MigrationProject project; private ChompbConfigService.ChompbConfig chompbConfig; private PermissionsService permissionsService; - private StreamingMetadataService streamingMetadataService; private PIDMinter pidMinter; private CdmToDestMapper cdmToDestMapper = new CdmToDestMapper(); private WorkGeneratorFactory workGeneratorFactory; @@ -113,7 +112,6 @@ private void initDependencies(SipGenerationOptions options, Connection conn) thr workGeneratorFactory.setPostMigrationReportService(postMigrationReportService); workGeneratorFactory.setAggregateTopMappingService(aggregateTopMappingService); workGeneratorFactory.setAggregateBottomMappingService(aggregateBottomMappingService); - workGeneratorFactory.setStreamingMetadataService(streamingMetadataService); try { workGeneratorFactory.setPermissionsInfo(permissionsService.loadMappings(project)); } catch (NoSuchFileException e) { @@ -445,10 +443,6 @@ public void setPermissionsService(PermissionsService permissionsService) { this.permissionsService = permissionsService; } - public void setStreamingMetadataService(StreamingMetadataService streamingMetadataService) { - this.streamingMetadataService = streamingMetadataService; - } - public void setPidMinter(PIDMinter pidMinter) { this.pidMinter = pidMinter; } diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFileService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFileService.java index 9200c5cb..927bc76c 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFileService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/SourceFileService.java @@ -20,7 +20,6 @@ import org.slf4j.Logger; import java.io.BufferedWriter; -import java.io.File; import java.io.IOException; import java.io.Reader; import java.nio.file.FileSystems; @@ -66,7 +65,6 @@ public class SourceFileService { protected MigrationProject project; protected CdmIndexService indexService; - protected StreamingMetadataService streamingMetadataService; public SourceFileService() { } @@ -108,11 +106,8 @@ public void generateMapping(GenerateSourceFileMappingOptions options) throws IOE // Generate source file mapping entry for each returned object while (rs.next()) { String cdmId = rs.getString(1); - // when creating a blank source files csv, filter out source files for duracloud content if (options.isPopulateBlank()) { - if (!streamingMetadataService.verifyRecordHasStreamingMetadata(cdmId)) { - csvPrinter.printRecord(cdmId, null, null, null); - } + csvPrinter.printRecord(cdmId, null, null, null); continue; } @@ -587,8 +582,4 @@ public void setProject(MigrationProject project) { public void setIndexService(CdmIndexService indexService) { this.indexService = indexService; } - - public void setStreamingMetadataService(StreamingMetadataService streamingMetadataService) { - this.streamingMetadataService = streamingMetadataService; - } } diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java deleted file mode 100644 index ce364610..00000000 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java +++ /dev/null @@ -1,134 +0,0 @@ -package edu.unc.lib.boxc.migration.cdm.services; - -import edu.unc.lib.boxc.migration.cdm.exceptions.MigrationException; -import edu.unc.lib.boxc.migration.cdm.model.CdmFieldInfo; -import edu.unc.lib.boxc.migration.cdm.model.MigrationProject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.List; - -/** - * Service for retrieving streaming metadata - * @author krwong - */ -public class StreamingMetadataService { - private static final Logger log = LoggerFactory.getLogger(StreamingMetadataService.class); - - private MigrationProject project; - private CdmFieldService fieldService; - private CdmIndexService indexService; - - public static final String STREAMING_FILE_FIELD = "stream"; - public static final String DURACLOUD_SPACE_FIELD = "duracl"; - public static final String PLAYLIST_FILE_EXTENSION = "-playlist.m3u8"; - public static final String DURACLOUD_OPEN = "open-hls"; - public static final String DURACLOUD_CAMPUS = "campus-hls"; - public static final String DURACLOUD_CLOSED = "closed-hls"; - public static final String STREAMING_HOST = "duracloud"; - - private Boolean projectHasStreamingMetadata = null; - - /** - * Verify if a record has streaming metadata - * @param cdmId - * @return true/false - */ - public boolean verifyRecordHasStreamingMetadata(String cdmId) { - if (!hasProjectStreamingMetadataField()) { - return false; - } - - var streamingFields = getStreamingFieldValues(cdmId); - return streamingFields[0] != null && streamingFields[1] != null; - } - - public boolean hasProjectStreamingMetadataField() { - if (projectHasStreamingMetadata == null) { - // check if project has streamingFile field and duracloudSpace field - fieldService.validateFieldsFile(project); - CdmFieldInfo fieldInfo = fieldService.loadFieldsFromProject(project); - List exportFields = fieldInfo.listAllExportFields(); - projectHasStreamingMetadata = exportFields.contains(STREAMING_FILE_FIELD) && - exportFields.contains(DURACLOUD_SPACE_FIELD); - } - return projectHasStreamingMetadata; - } - - /** - * Retrieve streaming metadata and remap to correct values - * @param cdmId - * @return object with streamingFile, duracloudSpace, and streamingHost fields - */ - public String[] getStreamingMetadata(String cdmId) { - var streamingValues = getStreamingFieldValues(cdmId); - String duracloudSpace = streamingValues[1]; - String streamingFile = streamingValues[0]; - - if (duracloudSpace == null || streamingFile == null) { - throw new MigrationException("Streaming metadata not found for " + cdmId); - } - // transform to playlist file extensions - streamingFile = streamingFile.split("\\.")[0] + PLAYLIST_FILE_EXTENSION; - - // transform to current duracloud space IDs - if (duracloudSpace.contains("open")) { - duracloudSpace = DURACLOUD_OPEN; - } else if (duracloudSpace.contains("campus")) { - duracloudSpace = DURACLOUD_CAMPUS; - } else if (duracloudSpace.contains("closed")) { - duracloudSpace = DURACLOUD_CLOSED; - } - - // original file extension (needed for streamingType) - String streamingFileExtension = streamingValues[0]; - - return new String[] {streamingFile, duracloudSpace, STREAMING_HOST, streamingFileExtension}; - } - - private String[] getStreamingFieldValues(String cdmId) { - String duracloudSpace = null; - String streamingFile = null; - - // retrieve streaming metadata - Connection conn = null; - try { - conn = indexService.openDbConnection(); - Statement stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery("select " + STREAMING_FILE_FIELD + ", " + DURACLOUD_SPACE_FIELD - + " from " + CdmIndexService.TB_NAME - + " where " + CdmFieldInfo.CDM_ID + " = '" + cdmId + "'"); - while (rs.next()) { - final String streamingVal = rs.getString(1); - if (streamingVal != null && !streamingVal.isEmpty()) { - streamingFile = streamingVal; - } - final String duracloudVal = rs.getString(2); - if (duracloudVal != null && !duracloudVal.isEmpty()) { - duracloudSpace = duracloudVal; - } - } - } catch (SQLException e) { - throw new MigrationException("Error interacting with export index", e); - } finally { - CdmIndexService.closeDbConnection(conn); - } - return new String[] {streamingFile, duracloudSpace}; - } - - public void setProject(MigrationProject project) { - this.project = project; - } - - public void setFieldService(CdmFieldService fieldService) { - this.fieldService = fieldService; - } - - public void setIndexService(CdmIndexService indexService) { - this.indexService = indexService; - } -} diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/sips/WorkGenerator.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/sips/WorkGenerator.java index 96aaed8e..306350aa 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/sips/WorkGenerator.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/sips/WorkGenerator.java @@ -17,14 +17,12 @@ import edu.unc.lib.boxc.migration.cdm.services.PostMigrationReportService; import edu.unc.lib.boxc.migration.cdm.services.RedirectMappingService; import edu.unc.lib.boxc.migration.cdm.services.SipService; -import edu.unc.lib.boxc.migration.cdm.services.StreamingMetadataService; import edu.unc.lib.boxc.model.api.DatastreamType; import edu.unc.lib.boxc.model.api.ids.PID; import edu.unc.lib.boxc.model.api.ids.PIDMinter; import edu.unc.lib.boxc.model.api.rdf.Cdr; import edu.unc.lib.boxc.model.api.rdf.CdrAspace; import edu.unc.lib.boxc.model.api.rdf.CdrDeposit; -import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.apache.jena.rdf.model.Bag; import org.apache.jena.rdf.model.Model; @@ -46,7 +44,6 @@ import static edu.unc.lib.boxc.auth.api.AccessPrincipalConstants.PUBLIC_PRINC; import static edu.unc.lib.boxc.migration.cdm.util.CLIConstants.outputLogger; import static edu.unc.lib.boxc.model.api.DatastreamType.ORIGINAL_FILE; -import static org.apache.jena.rdf.model.ResourceFactory.createProperty; import static org.slf4j.LoggerFactory.getLogger; /** @@ -56,11 +53,6 @@ */ public class WorkGenerator { private static final Logger log = getLogger(WorkGenerator.class); - // use local streamingUrl property for now because Cdr.streamingUrl only exists in a feature branch - public static final Property STREAMING_URL = createProperty( - "http://cdr.unc.edu/definitions/model#streamingUrl"); - public static final Property STREAMING_TYPE = createProperty( - "http://cdr.unc.edu/definitions/model#streamingType"); protected PIDMinter pidMinter; protected RedirectMappingService redirectMappingService; protected SourceFilesInfo sourceFilesInfo; @@ -78,7 +70,6 @@ public class WorkGenerator { protected AspaceRefIdService aspaceRefIdService; protected PostMigrationReportService postMigrationReportService; protected PermissionsInfo permissionsInfo; - protected StreamingMetadataService streamingMetadataService; protected MigrationProject project; protected String cdmId; @@ -165,14 +156,12 @@ protected Path getDescriptionPath(String cdmId, boolean allowMissing) { protected SourceFilesInfo.SourceFileMapping getSourceFileMapping(String cdmId) { SourceFilesInfo.SourceFileMapping sourceMapping = sourceFilesInfo.getMappingByCdmId(cdmId); if (sourceMapping == null || sourceMapping.getSourcePaths() == null) { - if (!streamingMetadataService.verifyRecordHasStreamingMetadata(cdmId)) { - String message = "Cannot transform object " + cdmId + ", no source file has been mapped"; - if (options.isForce()) { - outputLogger.info(message); - throw new SipService.SkipObjectException(); - } else { - throw new InvalidProjectStateException(message); - } + String message = "Cannot transform object " + cdmId + ", no source file has been mapped"; + if (options.isForce()) { + outputLogger.info(message); + throw new SipService.SkipObjectException(); + } else { + throw new InvalidProjectStateException(message); } } return sourceMapping; @@ -185,11 +174,10 @@ protected Resource makeFileResource(PID fileObjPid, Path sourcePath) { workBag.add(fileObjResc); // Link source file - if (sourcePath != null) { - Resource origResc = DepositModelHelpers.addDatastream(fileObjResc, ORIGINAL_FILE); - origResc.addLiteral(CdrDeposit.stagingLocation, sourcePath.toUri().toString()); - origResc.addLiteral(CdrDeposit.label, sourcePath.getFileName().toString()); - } + Resource origResc = DepositModelHelpers.addDatastream(fileObjResc, ORIGINAL_FILE); + origResc.addLiteral(CdrDeposit.stagingLocation, sourcePath.toUri().toString()); + origResc.addLiteral(CdrDeposit.label, sourcePath.getFileName().toString()); + return fileObjResc; } @@ -203,9 +191,6 @@ protected PID addFileObject(String cdmId, String cdmFileCreated, SourceFilesInfo // Add permission to source file addFilePermission(cdmId, fileObjResc); - // Add streamingUrl - addStreamingMetadata(cdmId, fileObjResc); - // Link access file if (accessFilesInfo != null) { SourceFilesInfo.SourceFileMapping accessMapping = accessFilesInfo.getMappingByCdmId(cdmId); @@ -269,24 +254,6 @@ protected void addPermission(String cdmId, Resource resource) { } } - protected void addStreamingMetadata(String cdmId, Resource resource) { - if (streamingMetadataService.verifyRecordHasStreamingMetadata(cdmId)) { - String[] streamingMetadata = streamingMetadataService.getStreamingMetadata(cdmId); - String duracloudSpace = streamingMetadata[1]; - String streamingFile = streamingMetadata[0]; - String streamingFileOriginalExtension = streamingMetadata[3]; - String streamingUrlValue = "https://durastream.lib.unc.edu/player?spaceId=" + duracloudSpace - + "&filename=" + streamingFile; - resource.addProperty(STREAMING_URL, streamingUrlValue); - // set streamingType to sound if mp3 and video if mp4 or anything else (for now) - if (FilenameUtils.getExtension(streamingFileOriginalExtension).equalsIgnoreCase("mp3")) { - resource.addProperty(STREAMING_TYPE, "sound"); - } else { - resource.addProperty(STREAMING_TYPE, "video"); - } - } - } - protected void addAltText(String cdmId, PID pid) throws IOException { if (altTextInfo != null) { AltTextInfo.AltTextMapping altTextMapping = altTextInfo.getMappingByCdmId(cdmId); diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/sips/WorkGeneratorFactory.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/sips/WorkGeneratorFactory.java index 33339e18..a9793ec3 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/sips/WorkGeneratorFactory.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/sips/WorkGeneratorFactory.java @@ -14,7 +14,6 @@ import edu.unc.lib.boxc.migration.cdm.services.DescriptionsService; import edu.unc.lib.boxc.migration.cdm.services.PostMigrationReportService; import edu.unc.lib.boxc.migration.cdm.services.RedirectMappingService; -import edu.unc.lib.boxc.migration.cdm.services.StreamingMetadataService; import edu.unc.lib.boxc.model.api.ids.PIDMinter; import java.io.IOException; @@ -44,7 +43,6 @@ public class WorkGeneratorFactory { private AggregateFileMappingService aggregateBottomMappingService; private PIDMinter pidMinter; private PermissionsInfo permissionsInfo; - private StreamingMetadataService streamingMetadataService; private MigrationProject project; public WorkGenerator create(String cdmId, String cdmCreated, String entryType) throws IOException { @@ -75,7 +73,6 @@ public WorkGenerator create(String cdmId, String cdmCreated, String entryType) t gen.redirectMappingService = redirectMappingService; gen.postMigrationReportService = postMigrationReportService; gen.permissionsInfo = permissionsInfo; - gen.streamingMetadataService = streamingMetadataService; gen.project = project; return gen; } @@ -152,10 +149,6 @@ public void setPermissionsInfo(PermissionsInfo permissionsInfo) { this.permissionsInfo = permissionsInfo; } - public void setStreamingMetadataService(StreamingMetadataService streamingMetadataService) { - this.streamingMetadataService = streamingMetadataService; - } - public void setProject(MigrationProject project) { this.project = project; } diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/status/ProjectStatusService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/status/ProjectStatusService.java index 783d35b3..8fdaeb9d 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/status/ProjectStatusService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/status/ProjectStatusService.java @@ -12,7 +12,6 @@ import edu.unc.lib.boxc.migration.cdm.services.CdmFieldService; import edu.unc.lib.boxc.migration.cdm.services.DescriptionsService; import edu.unc.lib.boxc.migration.cdm.services.SipService; -import edu.unc.lib.boxc.migration.cdm.services.StreamingMetadataService; import org.apache.commons.lang3.StringUtils; /** @@ -22,7 +21,6 @@ */ public class ProjectStatusService extends AbstractStatusService { private CdmFieldService fieldService; - private StreamingMetadataService streamingMetadataService; public void report() { outputLogger.info("Status for project {}", project.getProjectName()); @@ -118,7 +116,6 @@ private void reportSourceMappings(int totalObjects) { SourceFilesStatusService statusService = new SourceFilesStatusService(); statusService.setProject(project); statusService.setQueryService(getQueryService()); - statusService.setStreamingMetadataService(streamingMetadataService); statusService.reportStats(totalObjects, Verbosity.QUIET); } @@ -126,7 +123,6 @@ private void reportAccessMappings(int totalObjects) { AccessFilesStatusService statusService = new AccessFilesStatusService(); statusService.setProject(project); statusService.setQueryService(getQueryService()); - statusService.setStreamingMetadataService(streamingMetadataService); statusService.reportStats(totalObjects, Verbosity.QUIET); } @@ -146,8 +142,4 @@ private void reportDescriptionStats(int totalObjects) { descStatus.setQueryService(getQueryService()); descStatus.reportStats(totalObjects, Verbosity.QUIET); } - - public void setStreamingMetadataService(StreamingMetadataService streamingMetadataService) { - this.streamingMetadataService = streamingMetadataService; - } } diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/status/SourceFilesStatusService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/status/SourceFilesStatusService.java index 2c85931d..3f07fda6 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/status/SourceFilesStatusService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/status/SourceFilesStatusService.java @@ -14,7 +14,6 @@ import edu.unc.lib.boxc.migration.cdm.model.SourceFilesInfo; import edu.unc.lib.boxc.migration.cdm.model.SourceFilesInfo.SourceFileMapping; import edu.unc.lib.boxc.migration.cdm.options.Verbosity; -import edu.unc.lib.boxc.migration.cdm.services.StreamingMetadataService; import edu.unc.lib.boxc.migration.cdm.services.SourceFileService; import edu.unc.lib.boxc.migration.cdm.validators.SourceFilesValidator; import org.slf4j.Logger; @@ -27,8 +26,6 @@ public class SourceFilesStatusService extends AbstractStatusService { private static final Logger log = getLogger(SourceFilesStatusService.class); - private StreamingMetadataService streamingMetadataService; - /** * Display a stand alone report of the source file mapping status * @param verbosity @@ -52,7 +49,6 @@ public void reportStats(int totalObjects, Verbosity verbosity) { } SourceFilesValidator validator = getValidator(); validator.setProject(project); - validator.setStreamingMetadataService(streamingMetadataService); List errors = validator.validateMappings(forceValidation()); int numErrors = errors.size(); if (numErrors == 0) { @@ -121,10 +117,6 @@ protected SourceFileService getMappingService() { return new SourceFileService(); } - public void setStreamingMetadataService(StreamingMetadataService streamingMetadataService) { - this.streamingMetadataService = streamingMetadataService; - } - protected boolean forceValidation() { return false; } diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/validators/SourceFilesValidator.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/validators/SourceFilesValidator.java index 919e6e4f..8e8e3322 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/validators/SourceFilesValidator.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/validators/SourceFilesValidator.java @@ -12,7 +12,6 @@ import edu.unc.lib.boxc.migration.cdm.model.SourceFilesInfo.SourceFileMapping; import edu.unc.lib.boxc.migration.cdm.services.SourceFileService; -import edu.unc.lib.boxc.migration.cdm.services.StreamingMetadataService; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -30,7 +29,6 @@ */ public class SourceFilesValidator { protected MigrationProject project; - protected StreamingMetadataService streamingMetadataService; protected Set previousIds = new HashSet<>(); protected Set previousPaths = new HashSet<>(); protected List errors = new ArrayList<>(); @@ -77,7 +75,7 @@ public List validateMappings(boolean force) { protected void validateSourcePath(int i, String id, SourceFileMapping mapping, boolean force) { if (mapping.getSourcePaths() == null || mapping.getSourcePaths().isEmpty()) { - if (!force && !allowUnmapped() && !streamingMetadataService.verifyRecordHasStreamingMetadata(id)) { + if (!force && !allowUnmapped()) { errors.add("No path mapped at line " + i); } return; @@ -112,10 +110,6 @@ public void setProject(MigrationProject project) { this.project = project; } - public void setStreamingMetadataService(StreamingMetadataService streamingMetadataService) { - this.streamingMetadataService = streamingMetadataService; - } - protected boolean allowUnmapped() { return false; } diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/CompleteMigrationIT.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/CompleteMigrationIT.java index 24adc55d..c00be05f 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/CompleteMigrationIT.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/CompleteMigrationIT.java @@ -4,8 +4,6 @@ import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; -import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.STREAMING_TYPE; -import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.STREAMING_URL; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -338,96 +336,6 @@ public void migrateGroupedCollectionTest() throws Exception { assertDepositStatusSet(sip); } - @Test - public void fileWithStreamingOnlyNoSourceFileTest() throws Exception { - mockFieldInfoUrl("cdm_fields_resp.json", "mini_gilmer_duracloud"); - - String[] argsInit = new String[] { - "-w", baseDir.toString(), - "--env-config", chompbConfigPath, - "init", - "-p", "mini_gilmer_duracloud", - "-e", "test"}; - executeExpectSuccess(argsInit); - - Path projPath = baseDir.resolve("mini_gilmer_duracloud"); - MigrationProject project = new MigrationProject(projPath); - - String[] argsExport = new String[] { - "-w", projPath.toString(), - "--env-config", chompbConfigPath, - "export", - "-p", TestSshServer.PASSWORD }; - executeExpectSuccess(argsExport); - - String[] argsIndex = new String[] { - "-w", projPath.toString(), - "index"}; - executeExpectSuccess(argsIndex); - - String[] argsDest = new String[] { - "-w", projPath.toString(), - "destinations", "generate", - "-dd", DEST_UUID}; - executeExpectSuccess(argsDest); - - testHelper = new SipServiceHelper(project, filesBasePath); - Path sourcePath1 = testHelper.addSourceFile("276_182_E.tif"); - Path sourcePath2 = testHelper.addSourceFile("276_183_E.tif"); - // No source file item 27, so that it will only have streaming - - String[] argsSource = new String[] { - "-w", projPath.toString(), - "source_files", "generate", - "-b", testHelper.getSourceFilesBasePath().toString(), - "-n", "file"}; - executeExpectSuccess(argsSource); - - Files.copy(Paths.get("src/test/resources/mods_collections/gilmer_mods1.xml"), - project.getDescriptionsPath().resolve("gilmer_mods1.xml")); - String[] argsDesc = new String[] { - "-w", projPath.toString(), - "descriptions", "expand" }; - executeExpectSuccess(argsDesc); - - String[] args = new String[] { - "-w", projPath.toString(), - "sips", "generate" }; - executeExpectSuccess(args); - - MigrationSip sip = testHelper.extractSipFromOutput(output); - - DepositDirectoryManager dirManager = testHelper.createDepositDirectoryManager(sip); - Model model = testHelper.getSipModel(sip); - - Bag depBag = model.getBag(sip.getDepositPid().getRepositoryPath()); - List depBagChildren = depBag.iterator().toList(); - assertEquals(3, depBagChildren.size()); - - Resource workResc1 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-23"); - testHelper.assertObjectPopulatedInSip(workResc1, dirManager, model, sourcePath1, null, "25"); - // Work 2 has a source file and a streaming url - Resource workResc2 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-24"); - testHelper.assertObjectPopulatedInSip(workResc2, dirManager, model, sourcePath2, null, "26"); - Resource fileResc2 = testHelper.getFirstSipFileInWork(workResc2, dirManager, model); - assertTrue(fileResc2.hasProperty(STREAMING_URL)); - assertTrue(fileResc2.hasProperty(STREAMING_TYPE)); - // Work 3 has no source file, but does have a streaming url - Resource workResc3 = testHelper.getResourceByCreateTime(depBagChildren, "2005-12-08"); - testHelper.assertObjectPopulatedInSip(workResc3, dirManager, model, null, null, "27"); - Resource fileResc3 = testHelper.getFirstSipFileInWork(workResc3, dirManager, model); - assertTrue(fileResc3.hasProperty(STREAMING_URL)); - assertTrue(fileResc3.hasProperty(STREAMING_TYPE)); - - String[] argsSubmit = new String[] { - "-w", projPath.toString(), - "submit", - "-g", GROUPS }; - executeExpectSuccess(argsSubmit); - - assertDepositStatusSet(sip); - } - private String findFileIdByStagingLocation(Bag workBag, Path stagingLoc) { Resource fileResc = testHelper.findChildByStagingLocation(workBag, stagingLoc); return PIDs.get(fileResc.getURI()).getId(); diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/SipsCommandIT.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/SipsCommandIT.java index 6c4862b1..18067eaa 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/SipsCommandIT.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/SipsCommandIT.java @@ -21,8 +21,6 @@ import java.util.List; import java.util.stream.Stream; -import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.STREAMING_TYPE; -import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.STREAMING_URL; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -225,59 +223,6 @@ public void generateWithChildDescriptionsTest() throws Exception { assertChildFileModsPopulated(dirManager, workResc3, "27/original_file"); } - @Test - public void generateStreamingFilesOnlyTest() throws Exception { - testHelper.indexExportData("mini_gilmer_duracloud"); - testHelper.generateDefaultDestinationsMapping(DEST_UUID, null); - testHelper.populateDescriptions("gilmer_mods1.xml"); - List stagingLocs = testHelper.populateSourceFiles("276_182_E.tif", "276_183_E.tif", "276_203_E.tif"); - - String[] args = new String[] { - "-w", project.getProjectPath().toString(), - "sips", "generate" }; - executeExpectSuccess(args); - - MigrationSip sip = extractSipFromOutput(); - - DepositDirectoryManager dirManager = testHelper.createDepositDirectoryManager(sip); - Model model = testHelper.getSipModel(sip); - - Bag depBag = model.getBag(sip.getDepositPid().getRepositoryPath()); - List depBagChildren = depBag.iterator().toList(); - assertEquals(3, depBagChildren.size()); - - Resource workResc1 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-23"); - testHelper.assertObjectPopulatedInSip(workResc1, dirManager, model, stagingLocs.get(0), null, "25"); - Bag workResc1Bag = model.getBag(workResc1); - List workResc1Children = workResc1Bag.iterator().toList(); - assertEquals(1, workResc1Children.size()); - Resource workResc1FileObj = workResc1Children.get(0).asResource(); - assertFalse(workResc1FileObj.hasProperty(STREAMING_URL, "https://durastream.lib.unc.edu/player?" + - "spaceId=open-hls&filename=gilmer_recording-playlist.m3u8")); - - // mp4 file extension - Resource workResc2 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-24"); - testHelper.assertObjectPopulatedInSip(workResc2, dirManager, model, stagingLocs.get(1), null, "26"); - Bag workResc2Bag = model.getBag(workResc2); - List workResc2Children = workResc2Bag.iterator().toList(); - assertEquals(1, workResc2Children.size()); - Resource workResc2FileObj = workResc2Children.get(0).asResource(); - assertTrue(workResc2FileObj.hasProperty(STREAMING_URL, "https://durastream.lib.unc.edu/player?" + - "spaceId=open-hls&filename=gilmer_video-playlist.m3u8")); - assertTrue(workResc2FileObj.hasProperty(STREAMING_TYPE, "video")); - - // mp3 file extension - Resource workResc3 = testHelper.getResourceByCreateTime(depBagChildren, "2005-12-08"); - testHelper.assertObjectPopulatedInSip(workResc3, dirManager, model, stagingLocs.get(2), null, "27"); - Bag workResc3Bag = model.getBag(workResc3); - List workResc3Children = workResc3Bag.iterator().toList(); - assertEquals(1, workResc3Children.size()); - Resource workResc3FileObj = workResc3Children.get(0).asResource(); - assertTrue(workResc3FileObj.hasProperty(STREAMING_URL, "https://durastream.lib.unc.edu/player?" + - "spaceId=open-hls&filename=gilmer_recording-playlist.m3u8")); - assertTrue(workResc3FileObj.hasProperty(STREAMING_TYPE, "sound")); - } - @Test public void generateOneDestinationHasNoWorksTest() throws Exception { testHelper.indexExportData("mini_gilmer"); diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/SourceFilesCommandIT.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/SourceFilesCommandIT.java index 9c65cf2f..773650cd 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/SourceFilesCommandIT.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/SourceFilesCommandIT.java @@ -459,21 +459,6 @@ public void generateAddSourceFileVerboseTest() throws Exception { assertOutputContains("26,276_183_E.tif," + srcPath2); } - @Test - public void generateBlankStreamingMetadataSucceedsTest() throws Exception { - testHelper.indexExportData("mini_gilmer_duracloud"); - String[] args = new String[] { - "-w", project.getProjectPath().toString(), - "source_files", "generate", - "-B"}; - executeExpectSuccess(args); - - assertTrue(Files.exists(project.getSourceFilesMappingPath())); - assertOutputMatches(".*New Files Mapped: +0.*"); - assertOutputMatches(".*Total Files Mapped: +0.*"); - assertOutputMatches(".*Total Files in Project: +3.*"); - } - @Test public void validateValidTest() throws Exception { indexExportSamples(); @@ -518,28 +503,6 @@ public void validateInvalidTest() throws Exception { assertEquals(2, output.split(" - ").length, "Must only be two errors: " + output); } - - @Test - public void validateStreamingMetadataTest() throws Exception { - indexExportSamples(); - addSourceFile("276_182_E.tif"); - addSourceFile("276_183_E.tif"); - addSourceFile("276_203_E.tif"); - - String[] args = new String[] { - "-w", project.getProjectPath().toString(), - "source_files", "generate", - "-b", basePath.toString()}; - executeExpectSuccess(args); - - String[] args2 = new String[] { - "-w", project.getProjectPath().toString(), - "source_files", "validate" }; - executeExpectSuccess(args2); - - assertOutputContains("PASS: Source file mapping at path " + project.getSourceFilesMappingPath() + " is valid"); - } - @Test public void statusValidTest() throws Exception { indexExportSamples(); diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/StatusCommandIT.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/StatusCommandIT.java index d89685d2..a5b7c4e7 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/StatusCommandIT.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/StatusCommandIT.java @@ -2,7 +2,6 @@ import edu.unc.lib.boxc.migration.cdm.options.GenerateSourceFileMappingOptions; import edu.unc.lib.boxc.migration.cdm.options.SipGenerationOptions; -import edu.unc.lib.boxc.migration.cdm.options.SourceFileMappingOptions; import edu.unc.lib.boxc.migration.cdm.services.CdmFileRetrievalService; import edu.unc.lib.boxc.migration.cdm.services.SipService; import edu.unc.lib.boxc.migration.cdm.util.ProjectPropertiesSerialization; @@ -55,7 +54,7 @@ public void reportInitialized() throws Exception { assertOutputContains("CDM Collection Fields"); assertOutputMatches(".*Mapping File Valid: +Yes.*"); - assertOutputMatches(".*Fields: +63\n.*"); + assertOutputMatches(".*Fields: +61\n.*"); assertOutputMatches(".*Skipped: +1\n.*"); assertOutputContains("CDM Collection Exports"); diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/FieldAssessmentTemplateServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/FieldAssessmentTemplateServiceTest.java index 509fc5d4..7d14a8f9 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/FieldAssessmentTemplateServiceTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/FieldAssessmentTemplateServiceTest.java @@ -55,7 +55,7 @@ public void allExpectedCellsPopulatedTest() throws Exception { XSSFWorkbook workbook = new XSSFWorkbook(inputStream); Sheet sheet = workbook.getSheetAt(0); - assertEquals(62, sheet.getLastRowNum()); + assertEquals(60, sheet.getLastRowNum()); assertEquals(16, sheet.getRow(0).getPhysicalNumberOfCells()); assertEquals(12, sheet.getRow(1).getPhysicalNumberOfCells()); assertEquals(12, sheet.getRow(60).getPhysicalNumberOfCells()); diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/SipServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/SipServiceTest.java index 6af96bfb..a97d0704 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/SipServiceTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/SipServiceTest.java @@ -71,8 +71,6 @@ import static edu.unc.lib.boxc.auth.api.AccessPrincipalConstants.ON_CAMPUS_PRINC; import static edu.unc.lib.boxc.auth.api.AccessPrincipalConstants.PUBLIC_PRINC; import static edu.unc.lib.boxc.migration.cdm.services.CdmFieldService.CSV; -import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.STREAMING_TYPE; -import static edu.unc.lib.boxc.migration.cdm.services.sips.WorkGenerator.STREAMING_URL; import static edu.unc.lib.boxc.migration.cdm.test.PostMigrationReportTestHelper.assertContainsRow; import static java.nio.file.StandardOpenOption.APPEND; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -1513,59 +1511,6 @@ public void generateSipsGroupedWorkWorkPermissions() throws Exception { assertPersistedSipInfoMatches(sip); } - @Test - public void generateSipsWithStreamingUrl() throws Exception { - testHelper.indexExportData("mini_gilmer_duracloud"); - testHelper.generateDefaultDestinationsMapping(DEST_UUID, null); - testHelper.populateDescriptions("gilmer_mods1.xml"); - List stagingLocs = testHelper.populateSourceFiles("276_182_E.tif", "276_183_E.tif", "276_203_E.tif"); - - List sips = service.generateSips(makeOptions()); - assertEquals(1, sips.size()); - MigrationSip sip = sips.get(0); - - assertTrue(Files.exists(sip.getSipPath())); - - DepositDirectoryManager dirManager = testHelper.createDepositDirectoryManager(sip); - - Model model = testHelper.getSipModel(sip); - - Bag depBag = model.getBag(sip.getDepositPid().getRepositoryPath()); - List depBagChildren = depBag.iterator().toList(); - assertEquals(3, depBagChildren.size()); - - Resource workResc1 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-23"); - testHelper.assertObjectPopulatedInSip(workResc1, dirManager, model, stagingLocs.get(0), null, "25"); - Bag workResc1Bag = model.getBag(workResc1); - List workResc1Children = workResc1Bag.iterator().toList(); - assertEquals(1, workResc1Children.size()); - Resource workResc1FileObj = workResc1Children.get(0).asResource(); - assertFalse(workResc1FileObj.hasProperty(STREAMING_URL, "https://durastream.lib.unc.edu/player?" + - "spaceId=open-hls&filename=gilmer_recording-playlist.m3u8")); - - Resource workResc2 = testHelper.getResourceByCreateTime(depBagChildren, "2005-11-24"); - testHelper.assertObjectPopulatedInSip(workResc2, dirManager, model, stagingLocs.get(1), null, "26"); - Bag workResc2Bag = model.getBag(workResc2); - List workResc2Children = workResc2Bag.iterator().toList(); - assertEquals(1, workResc2Children.size()); - Resource workResc2FileObj = workResc2Children.get(0).asResource(); - assertTrue(workResc2FileObj.hasProperty(STREAMING_URL, "https://durastream.lib.unc.edu/player?" + - "spaceId=open-hls&filename=gilmer_video-playlist.m3u8")); - assertTrue(workResc2FileObj.hasProperty(STREAMING_TYPE, "video")); - - Resource workResc3 = testHelper.getResourceByCreateTime(depBagChildren, "2005-12-08"); - testHelper.assertObjectPopulatedInSip(workResc3, dirManager, model, stagingLocs.get(2), null, "27"); - Bag workResc3Bag = model.getBag(workResc3); - List workResc3Children = workResc3Bag.iterator().toList(); - assertEquals(1, workResc3Children.size()); - Resource workResc3FileObj = workResc3Children.get(0).asResource(); - assertTrue(workResc3FileObj.hasProperty(STREAMING_URL, "https://durastream.lib.unc.edu/player?" + - "spaceId=open-hls&filename=gilmer_recording-playlist.m3u8")); - assertTrue(workResc3FileObj.hasProperty(STREAMING_TYPE, "sound")); - - assertPersistedSipInfoMatches(sip); - } - @Test public void generateSipsWithAltText() throws Exception { testHelper.indexExportData("mini_gilmer"); diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/SourceFileServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/SourceFileServiceTest.java index b03b4d88..68dda580 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/SourceFileServiceTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/SourceFileServiceTest.java @@ -575,22 +575,6 @@ public void generateBlankTest() throws Exception { assertMappedDatePresent(); } - @Test - public void generateBlankStreamingMetadataTest() throws Exception { - testHelper.indexExportData("mini_gilmer_duracloud"); - GenerateSourceFileMappingOptions options = new GenerateSourceFileMappingOptions(); - options.setPopulateBlank(true); - - service.generateMapping(options); - - SourceFilesInfo info = service.loadMappings(); - assertMappingPresent(info, "25", "", null); - // 26 and 27 contain duracloud content and are excluded when generating blank source files - assertEquals(1, info.getMappings().size()); - - assertMappedDatePresent(); - } - @Test public void generateRespectsForceFlagTest() throws Exception { testHelper.indexExportData("mini_gilmer"); @@ -617,7 +601,8 @@ public void generateRespectsForceFlagTest() throws Exception { SourceFilesInfo info2 = service.loadMappings(); assertMappingPresent(info2, "25", "", null); - // 26 and 27 contain duracloud content and are excluded when generating blank source files + assertMappingPresent(info2, "26", "", null); + assertMappingPresent(info2, "27", "", null); assertEquals(3, info2.getMappings().size()); } diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java deleted file mode 100644 index f175e2d3..00000000 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java +++ /dev/null @@ -1,85 +0,0 @@ -package edu.unc.lib.boxc.migration.cdm.services; - -import edu.unc.lib.boxc.migration.cdm.exceptions.MigrationException; -import edu.unc.lib.boxc.migration.cdm.model.MigrationProject; -import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper; -import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper; -import edu.unc.lib.boxc.migration.cdm.test.SipServiceHelper; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.MockitoAnnotations.openMocks; - -public class StreamingMetadataServiceTest { - private static final String PROJECT_NAME = "proj"; - - @TempDir - public Path tmpFolder; - - private SipServiceHelper testHelper; - private MigrationProject project; - private StreamingMetadataService service; - private AutoCloseable closeable; - - @BeforeEach - public void setup() throws Exception { - closeable = openMocks(this); - project = MigrationProjectFactory.createMigrationProject( - tmpFolder, PROJECT_NAME, null, "user", CdmEnvironmentHelper.DEFAULT_ENV_ID, - BxcEnvironmentHelper.DEFAULT_ENV_ID, MigrationProject.PROJECT_SOURCE_CDM); - testHelper = new SipServiceHelper(project, tmpFolder); - service = new StreamingMetadataService(); - service.setProject(project); - service.setFieldService(testHelper.getFieldService()); - service.setIndexService(testHelper.getCdmIndexService()); - } - - @AfterEach - void closeService() throws Exception { - closeable.close(); - } - - @Test - public void verifyNoStreamingMetadata() throws Exception { - testHelper.indexExportData("mini_gilmer_duracloud"); - - var result = service.verifyRecordHasStreamingMetadata("25"); - assertFalse(result); - } - - @Test - public void verifyHasStreamingMetadata() throws Exception { - testHelper.indexExportData("mini_gilmer_duracloud"); - - var result = service.verifyRecordHasStreamingMetadata("27"); - assertTrue(result); - } - - @Test - public void getStreamingMetadataSuccess() throws Exception { - testHelper.indexExportData("mini_gilmer_duracloud"); - - var result = service.getStreamingMetadata("27"); - assertEquals("gilmer_recording-playlist.m3u8", result[0]); - assertEquals("open-hls", result[1]); - assertEquals("duracloud", result[2]); - assertEquals("gilmer_recording.mp3", result[3]); - } - - @Test - public void getStreamingMetadataFail() throws Exception { - testHelper.indexExportData("mini_gilmer_duracloud"); - - Exception exception = assertThrows(MigrationException.class, () -> { - service.getStreamingMetadata("25"); - }); - } -} diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/status/SourceFilesStatusServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/status/SourceFilesStatusServiceTest.java index f5b269ae..9c9ed88f 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/status/SourceFilesStatusServiceTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/status/SourceFilesStatusServiceTest.java @@ -43,7 +43,6 @@ public void setup() throws Exception { testHelper = new SipServiceHelper(project, tmpFolder); statusService = new SourceFilesStatusService(); statusService.setProject(project); - statusService.setStreamingMetadataService(testHelper.getStreamingMetadataService()); } @Test diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/test/SipServiceHelper.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/test/SipServiceHelper.java index 2cecc849..f8886890 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/test/SipServiceHelper.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/test/SipServiceHelper.java @@ -37,7 +37,6 @@ import edu.unc.lib.boxc.migration.cdm.services.FindingAidReportService; import edu.unc.lib.boxc.migration.cdm.services.GroupMappingService; import edu.unc.lib.boxc.migration.cdm.services.PermissionsService; -import edu.unc.lib.boxc.migration.cdm.services.StreamingMetadataService; import org.apache.commons.io.FileUtils; import org.apache.jena.rdf.model.Bag; import org.apache.jena.rdf.model.Model; @@ -105,7 +104,6 @@ public class SipServiceHelper { private FindingAidReportService findingAidReportService; private GroupMappingService groupMappingService; private PermissionsService permissionsService; - private StreamingMetadataService streamingMetadataService; private PIDMinter pidMinter; private PremisLoggerFactoryImpl premisLoggerFactory; private ChompbConfigService.ChompbConfig chompbConfig; @@ -130,18 +128,12 @@ public SipServiceHelper(MigrationProject project, Path filesBasePath) throws IOE fileIndexService = new FileIndexService(); fileIndexService.setProject(project); fileIndexService.setFieldService(fieldService); - streamingMetadataService = new StreamingMetadataService(); - streamingMetadataService.setProject(project); - streamingMetadataService.setFieldService(fieldService); - streamingMetadataService.setIndexService(cdmIndexService); sourceFileService = new SourceFileService(); sourceFileService.setIndexService(cdmIndexService); - sourceFileService.setStreamingMetadataService(streamingMetadataService); sourceFileService.setProject(project); accessFileService = new AccessFileService(); accessFileService.setIndexService(cdmIndexService); accessFileService.setProject(project); - accessFileService.setStreamingMetadataService(streamingMetadataService); altTextService = new AltTextService(); altTextService.setIndexService(cdmIndexService); altTextService.setProject(project); @@ -159,7 +151,6 @@ public SipServiceHelper(MigrationProject project, Path filesBasePath) throws IOE boxctronFileService = new BoxctronFileService(); boxctronFileService.setProject(project); boxctronFileService.setIndexService(cdmIndexService); - boxctronFileService.setStreamingMetadataService(streamingMetadataService); findingAidReportService = new FindingAidReportService(); findingAidReportService.setProject(project); findingAidReportService.setIndexService(cdmIndexService); @@ -185,7 +176,6 @@ public SipService createSipsService() { service.setAggregateTopMappingService(getAggregateFileMappingService()); service.setAggregateBottomMappingService(getAggregateBottomMappingService()); service.setPermissionsService(permissionsService); - service.setStreamingMetadataService(streamingMetadataService); return service; } @@ -606,7 +596,6 @@ public AggregateFileMappingService getAggregateFileMappingService() { this.aggregateFileMappingService = new AggregateFileMappingService(false); this.aggregateFileMappingService.setProject(project); this.aggregateFileMappingService.setIndexService(cdmIndexService); - this.aggregateFileMappingService.setStreamingMetadataService(streamingMetadataService); } return this.aggregateFileMappingService; } @@ -616,7 +605,6 @@ public AggregateFileMappingService getAggregateBottomMappingService() { this.aggregateBottomMappingService = new AggregateFileMappingService(true); this.aggregateBottomMappingService.setProject(project); this.aggregateBottomMappingService.setIndexService(cdmIndexService); - this.aggregateBottomMappingService.setStreamingMetadataService(streamingMetadataService); } return this.aggregateBottomMappingService; } @@ -664,10 +652,6 @@ public FileIndexService getFileIndexService() { return fileIndexService; } - public StreamingMetadataService getStreamingMetadataService() { - return streamingMetadataService; - } - public PIDMinter getPidMinter() { return pidMinter; } diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/validators/SourceFilesValidatorTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/validators/SourceFilesValidatorTest.java index b0a9df40..c01bbffa 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/validators/SourceFilesValidatorTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/validators/SourceFilesValidatorTest.java @@ -45,7 +45,6 @@ public void setup() throws Exception { testHelper = new SipServiceHelper(project, tmpFolder); validator = new SourceFilesValidator(); validator.setProject(project); - validator.setStreamingMetadataService(testHelper.getStreamingMetadataService()); } @Test @@ -214,27 +213,6 @@ public void ignorableErrorsWithForceTest() throws Exception { assertNumberErrors(errors, 0); } - @Test - public void streamingMetadataTest() throws Exception { - testHelper.indexExportData("mini_gilmer_duracloud"); - Path path2 = testHelper.addSourceFile("26.txt"); - writeCsv(mappingBody("26,," + path2 + ",", "27,,,")); - List errors = validator.validateMappings(false); - assertNumberErrors(errors, 0); - } - - @Test - public void streamingMetadataAndInvalidSourcePathTest() throws Exception { - testHelper.indexExportData("mini_gilmer"); - Path path = testHelper.addSourceFile("25.txt"); - Path path2 = testHelper.addSourceFile("27.txt"); - Files.delete(path2); - writeCsv(mappingBody("25,," + path + ",", "27,," + path2 + ",")); - List errors = validator.validateMappings(false); - assertHasError(errors, "Invalid path at line 3, file does not exist"); - assertNumberErrors(errors, 1); - } - private void assertHasError(List errors, String expected) { assertTrue(errors.contains(expected), "Expected error:\n" + expected + "\nBut the returned errors were:\n" + String.join("\n", errors)); diff --git a/src/test/resources/cdm_fields_resp.json b/src/test/resources/cdm_fields_resp.json index a6b3810a..c07bf73b 100644 --- a/src/test/resources/cdm_fields_resp.json +++ b/src/test/resources/cdm_fields_resp.json @@ -779,36 +779,6 @@ "admin": 0, "readonly": 0 }, - { - "name": "StreamingFile", - "nick": "stream", - "type": "TEXT", - "size": 0, - "find": "BLANK", - "req": 0, - "search": 0, - "hide": 1, - "vocdb": "", - "vocab": 1, - "dc": "BLANK", - "admin": 0, - "readonly": 0 - }, - { - "name": "duracloudSpace", - "nick": "duracl", - "type": "TEXT", - "size": 0, - "find": "e6", - "req": 0, - "search": 1, - "hide": 0, - "vocdb": "", - "vocab": 0, - "dc": "identi", - "admin": 0, - "readonly": 0 - }, { "name": "path", "nick": "full", diff --git a/src/test/resources/descriptions/03883-z/index/description/desc.all b/src/test/resources/descriptions/03883-z/index/description/desc.all index b45a606d..601b0d55 100644 --- a/src/test/resources/descriptions/03883-z/index/description/desc.all +++ b/src/test/resources/descriptions/03883-z/index/description/desc.all @@ -61,8 +61,6 @@ - - Volume1/03883_0005_0025.tif 231.jp2 @@ -136,8 +134,6 @@ - - Volume1/03883_0009_0051.tif 711.jp2 @@ -211,8 +207,6 @@ - - Volume1/03883_0009_0051.tif 711.jp2 diff --git a/src/test/resources/descriptions/03883/index/description/desc.all b/src/test/resources/descriptions/03883/index/description/desc.all index e452b4e6..08c693ac 100644 --- a/src/test/resources/descriptions/03883/index/description/desc.all +++ b/src/test/resources/descriptions/03883/index/description/desc.all @@ -61,8 +61,6 @@ - - Volume1/03883_0005_0025.tif 231.jp2 03883_folder_5 @@ -138,8 +136,6 @@ - - Volume1/03883_0009_0051.tif 711.jp2 03883_folder_9 @@ -215,8 +211,6 @@ - - Volume1/03883_0009_0051.tif 711.jp2 03883_folder_9 diff --git a/src/test/resources/descriptions/mini_gilmer_duracloud/index/description/desc.all b/src/test/resources/descriptions/mini_gilmer_duracloud/index/description/desc.all index 2d78e181..d4b65376 100644 --- a/src/test/resources/descriptions/mini_gilmer_duracloud/index/description/desc.all +++ b/src/test/resources/descriptions/mini_gilmer_duracloud/index/description/desc.all @@ -117,8 +117,6 @@ TIFF TIFF group1 -gilmer_video.mp4 -sfc20009-open /shc/gilmer_maps/ 276_183_E.tif 27.JP2 @@ -184,8 +182,6 @@ TIFF TIFF group2 -gilmer_recording.mp3 -sfc20009-open /shc/gilmer_maps/ 276_203_E.tif 50.jp2 diff --git a/src/test/resources/gilmer_fields.csv b/src/test/resources/gilmer_fields.csv index 7abd8ca3..81e39ac1 100644 --- a/src/test/resources/gilmer_fields.csv +++ b/src/test/resources/gilmer_fields.csv @@ -52,8 +52,6 @@ color,color,Color Space Raw Scan,false,n,n,y,n,BLANK coloa,coloa,Color Space filename,false,n,n,y,n,BLANK fila,fila,File Format Raw Scan,false,n,n,y,y,BLANK filb,filb,File Format filename,false,n,n,y,y,BLANK -stream,stream,StreamingFile,false,n,n,n,y,BLANK -duracl,duracl,duracloudSpace,false,n,n,n,y,identi full,full,path,false,n,n,n,n,BLANK fullrs,fullrs,Full resolution,false,n,n,y,n, dmoclcno,dmoclcno,OCLC number,false,n,n,y,n, diff --git a/src/test/resources/pdf_fields.csv b/src/test/resources/pdf_fields.csv index f1b685e2..fd409a68 100644 --- a/src/test/resources/pdf_fields.csv +++ b/src/test/resources/pdf_fields.csv @@ -61,8 +61,6 @@ creatc,creatc,Creator Identifier,false,n,y,n,n,identi search,search,Search by Decade,false,n,y,y,n,coverab pagina,pagina,Pagination,false,n,n,n,n,descri sort,sort,Sort Me,false,n,y,y,n,BLANK -stream,stream,StreamingFile,false,n,n,n,y,BLANK -duracl,duracl,duracloudSpace,false,n,n,n,y,identi datea,datea,Date of Recording,false,n,y,n,n,BLANK streaa,streaa,Streaming Host,false,n,y,n,y,BLANK record,record,Recording Location,false,n,y,n,y,subjec diff --git a/src/test/resources/roy_brown/cdm_fields.csv b/src/test/resources/roy_brown/cdm_fields.csv index 03c93841..9a35f91a 100644 --- a/src/test/resources/roy_brown/cdm_fields.csv +++ b/src/test/resources/roy_brown/cdm_fields.csv @@ -61,8 +61,6 @@ creatc,creatc,Creator Identifier,false,n,y,n,n,identi search,search,Search by Decade,false,n,y,y,n,coverab pagina,pagina,Pagination,false,n,n,n,n,descri sort,sort,Sort Me,false,n,y,y,n,BLANK -stream,stream,StreamingFile,false,n,n,n,y,BLANK -duracl,duracl,duracloudSpace,false,n,n,n,y,identi fullrs,fullrs,Archival file,false,n,n,y,n,BLANK dmoclcno,dmoclcno,OCLC number,false,n,n,y,n,BLANK dmcreated,dmcreated,Date created,false,y,n,y,n,BLANK