diff --git a/packages/core/src/actions/filesystem_actions.ts b/packages/core/src/actions/filesystem_actions.ts index 988d399f..70bee5d2 100644 --- a/packages/core/src/actions/filesystem_actions.ts +++ b/packages/core/src/actions/filesystem_actions.ts @@ -150,7 +150,9 @@ class FileSystemActions extends Actions { } #is_glob(globpath: string): boolean { - if (!path.isGlob(globpath)) { + // isGlob treats "\" as an escape character, but on windows it is the path separator (e.g. "C:\media\*.jpg") + const glob_check_path = Deno.build.os === 'windows' ? globpath.replaceAll('\\', '/') : globpath + if (!path.isGlob(glob_check_path)) { return false } if (globpath.includes('?') && !globpath.includes('*')) { diff --git a/packages/core/src/actions/ingest_actions.ts b/packages/core/src/actions/ingest_actions.ts index d3fc5301..a0a4fa51 100644 --- a/packages/core/src/actions/ingest_actions.ts +++ b/packages/core/src/actions/ingest_actions.ts @@ -130,7 +130,9 @@ class IngestActions extends Actions { return { stats } } - async #add_media_file(ctx: plugin.FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList): Promise { + async #add_media_file(ctx: plugin.FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, editing?: plugin.ReceiverEditing): Promise { + const create_editing = editing?.editor !== undefined ? { editor: editing.editor } : undefined + if (ctx.default_metadata?.media_info) { media_info = {...ctx.default_metadata.media_info as inputs.MediaInfo, ...media_info} } @@ -143,7 +145,7 @@ class IngestActions extends Actions { const existing_media = ctx.forager.media.get({filepath}) if (media_info || tags) { // const media_file = this.models.MediaFile.select_one({filepath: e.filepath}, {or_raise: true}) - ctx.forager.media.update(existing_media.media_reference.id, media_info, tags) + ctx.forager.media.update(existing_media.media_reference.id, media_info, tags, editing) ctx.logger.info(`Updated existing file ${ctx.entry.path}`) return {media_reference_id: existing_media.media_reference.id, status: 'updated'} } else { @@ -156,7 +158,7 @@ class IngestActions extends Actions { } try { - const created_media = await ctx.forager.media.create(filepath, media_info, tags, {editor: ctx.receiver.name}) + const created_media = await ctx.forager.media.create(filepath, media_info, tags, create_editing) return {media_reference_id: created_media.media_reference.id, checksum: created_media.media_file.checksum, status: 'created'} } catch (e) { let file_identifier = filepath @@ -176,11 +178,11 @@ class IngestActions extends Actions { if (e instanceof errors.MediaAlreadyExistsError) { const {media_reference} = ctx.forager.media.get({filepath: e.filepath}) // const media_file = this.models.MediaFile.select_one({filepath: e.filepath}, {or_raise: true}) - const updated_media = ctx.forager.media.update(media_reference.id, media_info, tags, {editor: ctx.receiver.name}) + const updated_media = ctx.forager.media.update(media_reference.id, media_info, tags, editing) return {media_reference_id: updated_media.media_reference.id, status: 'updated', checksum: updated_media.media_file.checksum} } else if (e instanceof errors.SeriesAlreadyExistsError) { const {media_reference} = ctx.forager.series.get({series_name: e.media_series_name}) - const updated_media = ctx.forager.series.update(media_reference.id, media_info, tags, {editor: ctx.receiver.name}) + const updated_media = ctx.forager.series.update(media_reference.id, media_info, tags, editing) return {media_reference_id: updated_media.media_reference.id, status: 'updated' } } else { throw new Error(`unexpected code path for error ${e.constructor.name}: ${e}`) @@ -225,8 +227,8 @@ class IngestActions extends Actions { } } - async #add(ctx: plugin.FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, series?: inputs.MediaSeriesBulk) { - const media_file_ack = await this.#add_media_file(ctx, filepath, media_info, tags) + async #add(ctx: plugin.FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, series?: inputs.MediaSeriesBulk, editing?: plugin.ReceiverEditing) { + const media_file_ack = await this.#add_media_file(ctx, filepath, media_info, tags, editing) this.models.FilesystemPath.update({ id: ctx.file_id, diff --git a/packages/core/src/actions/lib/base.ts b/packages/core/src/actions/lib/base.ts index 3b20bc53..6f7516ae 100644 --- a/packages/core/src/actions/lib/base.ts +++ b/packages/core/src/actions/lib/base.ts @@ -79,11 +79,16 @@ export interface MediaGroupResponse { export interface EditInfo { + /** Identifier of the editor, used to track which editor wrote a media info field or tag */ editor?: string } export interface CreateEditor extends EditInfo {} export interface UpdateEditor extends EditInfo { + /** + * Overwrite media info fields and tags written by other editors (including tags written without an editor). + * Defaults to false when an editor is supplied. Without an editor, updates always overwrite. + */ overwrite?: boolean } @@ -198,8 +203,8 @@ abstract class Actions extends Emitter { const media_info_updates = {...parsed.media_info} - // when overwrite is false, we will only overwrite fields that are already owned by this editor (or by no editors at all) - if (parsed.editing?.overwrite === false) { + // unless overwrite is set, an editor will only overwrite fields that are already owned by this editor (or by no editors at all) + if (parsed.editing?.editor !== undefined && parsed.editing.overwrite !== true) { const media_info_last_editors = this.models.EditLog.get_media_info_last_editors({media_reference_id}) const media_info_update_fields = Object.keys(media_info_updates) as (keyof outputs.MediaInfo)[] for (const media_info_field of media_info_update_fields) { @@ -370,7 +375,7 @@ abstract class Actions extends Emitter() const tags_existing = new Map() const editor = editing?.editor - const overwrite_edits_by_others = editing && 'overwrite' in editing && editing.overwrite + const overwrite_edits_by_others = editor === undefined || (editing !== undefined && 'overwrite' in editing && editing.overwrite === true) const existing_tags = this.models.Tag.select_all({media_reference_id: media_reference_id}) for (const tag of existing_tags) { @@ -380,7 +385,7 @@ abstract class Actions extends Emitter { if (editing?.overwrite === false && editing.editor === undefined) { diff --git a/packages/core/src/lib/plugin_script.ts b/packages/core/src/lib/plugin_script.ts index 942ec923..19223eaa 100644 --- a/packages/core/src/lib/plugin_script.ts +++ b/packages/core/src/lib/plugin_script.ts @@ -21,7 +21,18 @@ export interface FileSystemReceiverContext { entry: fs.WalkEntry forager: Forager logger: Logger - add(ctx: FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, series?: inputs.MediaSeriesBulk): Promise + add(ctx: FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, series?: inputs.MediaSeriesBulk, editing?: ReceiverEditing): Promise +} + +/** + * Editing options for media added by a {@linkcode FileSystemReceiver}. These are passed through to media create/update calls as-is. + * When omitted, no editing options are sent and the `editing` block from the forager config applies. + */ +export interface ReceiverEditing { + /** Identifier of the editor, can be used to distinguish where a particular media metadata field or tag came from */ + editor?: string + /** Overwrite edits of other editors when updating existing media. Defaults to false when an editor is supplied, so only tags and fields previously written by this editor are replaced */ + overwrite?: boolean } export abstract class FileSystemReceiver { @@ -32,8 +43,8 @@ export abstract class FileSystemReceiver { public abstract foreach(ctx: FileSystemReceiverContext): Promise - protected async add(ctx: FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, series?: inputs.MediaSeriesBulk) { - return ctx.add(ctx, filepath, media_info, tags, series) + protected async add(ctx: FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, series?: inputs.MediaSeriesBulk, editing?: ReceiverEditing) { + return ctx.add(ctx, filepath, media_info, tags, series, editing) } /** @internal */ diff --git a/packages/core/test/edit_log.test.ts b/packages/core/test/edit_log.test.ts index a54f5a19..04a52205 100644 --- a/packages/core/test/edit_log.test.ts +++ b/packages/core/test/edit_log.test.ts @@ -147,3 +147,51 @@ test('media edit log', async (ctx) => { ctx.assert.equals(media_generated_art.media_reference.source_created_at, source_created_at) }) }) + +test('editor replace ownership', async (ctx) => { + using forager = new Forager(ctx.get_test_config()) + forager.init() + const by_slug = (a: {group: string, name: string}, b: {group: string, name: string}) => `${a.group}:${a.name}`.localeCompare(`${b.group}:${b.name}`) + + const media = await forager.media.create(ctx.resources.media_files['koch.tif'], {}, ['no_editor_tag']) + const id = media.media_reference.id + forager.media.update(id, {title: 'bar title'}, {add: ['bar_tag']}, {editor: 'bar'}) + + await ctx.subtest('replace with an editor keeps tags from other editors and tags without an editor', () => { + const updated = forager.media.update(id, {}, ['my:tag'], {editor: 'foo'}) + ctx.assert.list_partial(updated.tags, [ + {group: '', name: 'bar_tag'}, + {group: '', name: 'no_editor_tag'}, + {group: 'my', name: 'tag'}, + ], by_slug) + ctx.assert.equals(updated.edit_log![0].changes.tags, {added: ['my:tag'], removed: []}) + }) + + await ctx.subtest('replace with an editor replaces tags owned by that editor', () => { + const updated = forager.media.update(id, {}, ['my:other_tag'], {editor: 'foo', overwrite: false}) + ctx.assert.list_partial(updated.tags, [ + {group: '', name: 'bar_tag'}, + {group: '', name: 'no_editor_tag'}, + {group: 'my', name: 'other_tag'}, + ], by_slug) + ctx.assert.equals(updated.edit_log![0].changes.tags, {added: ['my:other_tag'], removed: ['my:tag']}) + }) + + await ctx.subtest('an editor does not overwrite media info fields owned by other editors by default', () => { + const updated = forager.media.update(id, {title: 'foo title', description: 'foo description'}, undefined, {editor: 'foo'}) + ctx.assert.equals(updated.media_reference.title, 'bar title') + ctx.assert.equals(updated.media_reference.description, 'foo description') + }) + + await ctx.subtest('overwrite replaces tags and media info from all editors', () => { + const updated = forager.media.update(id, {title: 'foo title'}, ['my:tag'], {editor: 'foo', overwrite: true}) + ctx.assert.equals(updated.media_reference.title, 'foo title') + ctx.assert.list_partial(updated.tags, [{group: 'my', name: 'tag'}]) + }) + + await ctx.subtest('replace without an editor replaces all tags', () => { + forager.media.update(id, {}, {add: ['bar_tag']}, {editor: 'bar'}) + const updated = forager.media.update(id, {}, ['no_editor_tag']) + ctx.assert.list_partial(updated.tags, [{group: '', name: 'no_editor_tag'}]) + }) +}) diff --git a/packages/core/test/ingest.test.ts b/packages/core/test/ingest.test.ts index 3eb8d350..fa563641 100644 --- a/packages/core/test/ingest.test.ts +++ b/packages/core/test/ingest.test.ts @@ -1,6 +1,6 @@ import { test } from 'forager-test' import * as path from '@std/path' -import { Forager, type MediaFileResponse } from '~/mod.ts' +import { Forager, FileSystemReceiver, PluginScript, type FileSystemReceiverContext, type ReceiverEditing, type MediaFileResponse } from '~/mod.ts' test('ingest actions', async ctx => { using forager = new Forager(ctx.get_test_config()) @@ -38,6 +38,128 @@ test('ingest actions', async ctx => { }) +class TifReceiver extends FileSystemReceiver { + override name = 'tif_scraper' + override extensions = ['tif'] + tags: string[] = [] + editing?: ReceiverEditing + override foreach = async (ctx: FileSystemReceiverContext) => { + await this.add(ctx, ctx.entry.path, {title: 'scraped'}, this.tags, undefined, this.editing) + } +} + +test('ingest editing', async ctx => { + const receiver = new TifReceiver() + const plugin_script = new PluginScript() + plugin_script.recievers.unshift(receiver) + using forager = new Forager(ctx.get_test_config(), plugin_script) + forager.init() + + const filepath = ctx.resources.media_files['koch.tif'] + const tif_glob = ctx.resources.resources_directory + path.SEPARATOR + '*.tif' + const by_name = (a: {name: string}, b: {name: string}) => a.name.localeCompare(b.name) + + const manual_media = await forager.media.create(filepath, {}, ['manual_tag'], {editor: 'manual'}) + const media_reference_id = manual_media.media_reference.id + + await ctx.subtest('ingest over existing media preserves tags from other editors', async () => { + receiver.tags = ['scraper_1'] + receiver.editing = {editor: 'tif_scraper', overwrite: false} + await forager.filesystem.discover({path: tif_glob}) + const ingest = await forager.ingest.start() + ctx.assert.equals(ingest.stats.updated, 1) + + const media = forager.media.get({media_reference_id}) as MediaFileResponse + ctx.assert.equals(media.media_reference.title, 'scraped') + ctx.assert.list_partial(media.tags, [{name: 'manual_tag'}, {name: 'scraper_1'}], by_name) + ctx.assert.equals(media.edit_log![0].editor, 'tif_scraper') + ctx.assert.equals(media.edit_log![0].changes.tags, {added: ['scraper_1'], removed: []}) + }) + + await ctx.subtest('reingest only replaces tags owned by the receiver', async () => { + receiver.tags = ['scraper_2'] + await forager.filesystem.discover({path: tif_glob, reingest: true}) + await forager.ingest.start() + + const media = forager.media.get({media_reference_id}) as MediaFileResponse + ctx.assert.list_partial(media.tags, [{name: 'manual_tag'}, {name: 'scraper_2'}], by_name) + ctx.assert.equals(media.edit_log![0].editor, 'tif_scraper') + ctx.assert.equals(media.edit_log![0].changes.tags, {added: ['scraper_2'], removed: ['scraper_1']}) + }) + + await ctx.subtest('a different editor keeps tags from the previous receiver editor', async () => { + receiver.tags = ['custom_1'] + receiver.editing = {editor: 'custom_editor', overwrite: false} + await forager.filesystem.discover({path: tif_glob, reingest: true}) + await forager.ingest.start() + + const media = forager.media.get({media_reference_id}) as MediaFileResponse + ctx.assert.list_partial(media.tags, [{name: 'custom_1'}, {name: 'manual_tag'}, {name: 'scraper_2'}], by_name) + ctx.assert.equals(media.edit_log![0].editor, 'custom_editor') + }) + + await ctx.subtest('receiver can overwrite edits by others', async () => { + receiver.tags = ['overwrite_1'] + receiver.editing = {editor: 'tif_scraper', overwrite: true} + await forager.filesystem.discover({path: tif_glob, reingest: true}) + await forager.ingest.start() + + const media = forager.media.get({media_reference_id}) as MediaFileResponse + ctx.assert.list_partial(media.tags, [{name: 'overwrite_1'}]) + ctx.assert.equals(media.edit_log![0].editor, 'tif_scraper') + }) + + await ctx.subtest('receiver without editing options does not record an editor', async () => { + receiver.tags = ['no_editor_1'] + receiver.editing = undefined + const edit_log_count = (forager.media.get({media_reference_id}) as MediaFileResponse).edit_log!.length + await forager.filesystem.discover({path: tif_glob, reingest: true}) + await forager.ingest.start() + + const media = forager.media.get({media_reference_id}) as MediaFileResponse + ctx.assert.list_partial(media.tags, [{name: 'no_editor_1'}]) + ctx.assert.equals(media.edit_log!.length, edit_log_count) + }) +}) + +test('ingest editing on created media', async ctx => { + const receiver = new TifReceiver() + receiver.tags = ['created_tag'] + receiver.editing = {editor: 'custom_editor'} + const plugin_script = new PluginScript() + plugin_script.recievers.unshift(receiver) + using forager = new Forager(ctx.get_test_config(), plugin_script) + forager.init() + + await forager.filesystem.discover({path: ctx.resources.resources_directory + path.SEPARATOR + '*.tif'}) + const ingest = await forager.ingest.start() + ctx.assert.equals(ingest.stats.created, 1) + + const media = forager.media.get({filepath: ctx.resources.media_files['koch.tif']}) as MediaFileResponse + ctx.assert.equals(media.media_reference.editors, ['custom_editor']) + ctx.assert.list_partial(media.tags, [{name: 'created_tag'}]) + ctx.assert.list_partial(media.edit_log!, [{editor: 'custom_editor', operation_type: 'CREATE'}]) +}) + +test('ingest without editing on created media', async ctx => { + const receiver = new TifReceiver() + receiver.tags = ['created_tag'] + const plugin_script = new PluginScript() + plugin_script.recievers.unshift(receiver) + using forager = new Forager(ctx.get_test_config(), plugin_script) + forager.init() + + await forager.filesystem.discover({path: ctx.resources.resources_directory + path.SEPARATOR + '*.tif'}) + const ingest = await forager.ingest.start() + ctx.assert.equals(ingest.stats.created, 1) + + const media = forager.media.get({filepath: ctx.resources.media_files['koch.tif']}) as MediaFileResponse + ctx.assert.equals(media.media_reference.editors, null) + ctx.assert.list_partial(media.tags, [{name: 'created_tag'}]) + ctx.assert.equals(media.edit_log!.length, 0) +}) + + /* test('filesystem discovery', async (ctx) => { using forager = new Forager(ctx.get_test_config())