impl(core): editing argument on FileSystemReceiver add, editor-scoped tag replace

packages/core/src/actions/filesystem_actions.ts

@@ -150,7 +150,9 @@ class FileSystemActions extends Actions {
   }
 
   #is_glob(globpath: string): boolean {
-    if (!path.isGlob(globpath)) {
+    // isGlob treats "\" as an escape character, but on windows it is the path separator (e.g. "C:\media\*.jpg")
+    const glob_check_path = Deno.build.os === 'windows' ? globpath.replaceAll('\\', '/') : globpath
+    if (!path.isGlob(glob_check_path)) {
       return false
     }
     if (globpath.includes('?') && !globpath.includes('*')) {

packages/core/src/actions/ingest_actions.ts

@@ -130,7 +130,9 @@ class IngestActions extends Actions<IngestEvents> {
     return { stats }
   }
 
-  async #add_media_file(ctx: plugin.FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList): Promise<AddMediaFileAck> {
+  async #add_media_file(ctx: plugin.FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, editing?: plugin.ReceiverEditing): Promise<AddMediaFileAck> {
+    const create_editing = editing?.editor !== undefined ? { editor: editing.editor } : undefined
+
     if (ctx.default_metadata?.media_info) {
       media_info = {...ctx.default_metadata.media_info as inputs.MediaInfo, ...media_info}
     }
@@ -143,7 +145,7 @@ class IngestActions extends Actions<IngestEvents> {
       const existing_media = ctx.forager.media.get({filepath})
       if (media_info || tags) {
         // const media_file = this.models.MediaFile.select_one({filepath: e.filepath}, {or_raise: true})
-        ctx.forager.media.update(existing_media.media_reference.id, media_info, tags)
+        ctx.forager.media.update(existing_media.media_reference.id, media_info, tags, editing)
         ctx.logger.info(`Updated existing file ${ctx.entry.path}`)
         return {media_reference_id: existing_media.media_reference.id, status: 'updated'}
       } else {
@@ -156,7 +158,7 @@ class IngestActions extends Actions<IngestEvents> {
     }
 
     try {
-      const created_media = await ctx.forager.media.create(filepath, media_info, tags, {editor: ctx.receiver.name})
+      const created_media = await ctx.forager.media.create(filepath, media_info, tags, create_editing)
       return {media_reference_id: created_media.media_reference.id, checksum: created_media.media_file.checksum, status: 'created'}
     } catch (e) {
       let file_identifier = filepath
@@ -176,11 +178,11 @@ class IngestActions extends Actions<IngestEvents> {
         if (e instanceof errors.MediaAlreadyExistsError) {
           const {media_reference} = ctx.forager.media.get({filepath: e.filepath})
           // const media_file = this.models.MediaFile.select_one({filepath: e.filepath}, {or_raise: true})
-          const updated_media = ctx.forager.media.update(media_reference.id, media_info, tags, {editor: ctx.receiver.name})
+          const updated_media = ctx.forager.media.update(media_reference.id, media_info, tags, editing)
           return {media_reference_id: updated_media.media_reference.id, status: 'updated', checksum: updated_media.media_file.checksum}
         } else if (e instanceof errors.SeriesAlreadyExistsError) {
           const {media_reference} = ctx.forager.series.get({series_name: e.media_series_name})
-          const updated_media = ctx.forager.series.update(media_reference.id, media_info, tags, {editor: ctx.receiver.name})
+          const updated_media = ctx.forager.series.update(media_reference.id, media_info, tags, editing)
           return {media_reference_id: updated_media.media_reference.id, status: 'updated' }
         } else {
           throw new Error(`unexpected code path for error ${e.constructor.name}: ${e}`)
@@ -225,8 +227,8 @@ class IngestActions extends Actions<IngestEvents> {
       }
   }
 
-  async #add(ctx: plugin.FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, series?: inputs.MediaSeriesBulk) {
-    const media_file_ack = await this.#add_media_file(ctx, filepath, media_info, tags)
+  async #add(ctx: plugin.FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, series?: inputs.MediaSeriesBulk, editing?: plugin.ReceiverEditing) {
+    const media_file_ack = await this.#add_media_file(ctx, filepath, media_info, tags, editing)
 
     this.models.FilesystemPath.update({
       id: ctx.file_id,

packages/core/src/actions/lib/base.ts

@@ -79,11 +79,16 @@ export interface MediaGroupResponse {
 
 
 export interface EditInfo {
+  /** Identifier of the editor, used to track which editor wrote a media info field or tag */
   editor?: string
 }
 
 export interface CreateEditor extends EditInfo {}
 export interface UpdateEditor extends EditInfo {
+  /**
+   * Overwrite media info fields and tags written by other editors (including tags written without an editor).
+   * Defaults to false when an editor is supplied. Without an editor, updates always overwrite.
+   */
   overwrite?: boolean
 }
 
@@ -198,8 +203,8 @@ abstract class Actions<Events extends EmitterEvents = {}> extends Emitter<Events
     const transaction = this.ctx.db.transaction_sync(() => {
       const media_info_updates = {...parsed.media_info}
 
-      // when overwrite is false, we will only overwrite fields that are already owned by this editor (or by no editors at all)
-      if (parsed.editing?.overwrite === false) {
+      // unless overwrite is set, an editor will only overwrite fields that are already owned by this editor (or by no editors at all)
+      if (parsed.editing?.editor !== undefined && parsed.editing.overwrite !== true) {
         const media_info_last_editors = this.models.EditLog.get_media_info_last_editors({media_reference_id})
         const media_info_update_fields = Object.keys(media_info_updates) as (keyof outputs.MediaInfo)[]
         for (const media_info_field of media_info_update_fields) {
@@ -370,7 +375,7 @@ abstract class Actions<Events extends EmitterEvents = {}> extends Emitter<Events
     const tags_removed = new Set<string>()
     const tags_existing = new Map<string, TagJoin>()
     const editor = editing?.editor
-    const overwrite_edits_by_others = editing && 'overwrite' in editing && editing.overwrite
+    const overwrite_edits_by_others = editor === undefined || (editing !== undefined && 'overwrite' in editing && editing.overwrite === true)
 
     const existing_tags = this.models.Tag.select_all({media_reference_id: media_reference_id})
     for (const tag of existing_tags) {
@@ -380,7 +385,7 @@ abstract class Actions<Events extends EmitterEvents = {}> extends Emitter<Events
     if (tags.replace) {
       // remove all existing tags matching the editing stragegy first
       for (const [tag_slug, tag] of tags_existing.entries()) {
-        if (editor === undefined || tag.editor === null || tag.editor === editor || overwrite_edits_by_others) {
+        if (tag.editor === editor || overwrite_edits_by_others) {
           this.models.MediaReferenceTag.delete({media_reference_id, tag_id: tag.id})
           tags_removed.add(tag_slug)
         }

packages/core/src/lib/inputs_base.ts

@@ -23,7 +23,7 @@ export const CreateEditing = Editing.extend({
 
 
 export const UpdateEditing = Editing.extend({
-  overwrite: z.boolean().default(true)
+  overwrite: z.boolean().optional()
 }).optional()
 .refine(editing => {
   if (editing?.overwrite === false && editing.editor === undefined) {

packages/core/src/lib/plugin_script.ts

@@ -21,7 +21,18 @@ export interface FileSystemReceiverContext {
   entry: fs.WalkEntry
   forager: Forager
   logger: Logger
-  add(ctx: FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, series?: inputs.MediaSeriesBulk): Promise<void>
+  add(ctx: FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, series?: inputs.MediaSeriesBulk, editing?: ReceiverEditing): Promise<void>
+}
+
+/**
+ * Editing options for media added by a {@linkcode FileSystemReceiver}. These are passed through to media create/update calls as-is.
+ * When omitted, no editing options are sent and the `editing` block from the forager config applies.
+ */
+export interface ReceiverEditing {
+  /** Identifier of the editor, can be used to distinguish where a particular media metadata field or tag came from */
+  editor?: string
+  /** Overwrite edits of other editors when updating existing media. Defaults to false when an editor is supplied, so only tags and fields previously written by this editor are replaced */
+  overwrite?: boolean
 }
 
 export abstract class FileSystemReceiver {
@@ -32,8 +43,8 @@ export abstract class FileSystemReceiver {
 
   public abstract foreach(ctx: FileSystemReceiverContext): Promise<void>
 
-  protected async add(ctx: FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, series?: inputs.MediaSeriesBulk) {
-    return ctx.add(ctx, filepath, media_info, tags, series)
+  protected async add(ctx: FileSystemReceiverContext, filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.TagList, series?: inputs.MediaSeriesBulk, editing?: ReceiverEditing) {
+    return ctx.add(ctx, filepath, media_info, tags, series, editing)
   }
 
   /** @internal */

packages/core/test/edit_log.test.ts

@@ -147,3 +147,51 @@ test('media edit log', async (ctx) => {
     ctx.assert.equals(media_generated_art.media_reference.source_created_at, source_created_at)
   })
 })
+
+test('editor replace ownership', async (ctx) => {
+  using forager = new Forager(ctx.get_test_config())
+  forager.init()
+  const by_slug = (a: {group: string, name: string}, b: {group: string, name: string}) => `${a.group}:${a.name}`.localeCompare(`${b.group}:${b.name}`)
+
+  const media = await forager.media.create(ctx.resources.media_files['koch.tif'], {}, ['no_editor_tag'])
+  const id = media.media_reference.id
+  forager.media.update(id, {title: 'bar title'}, {add: ['bar_tag']}, {editor: 'bar'})
+
+  await ctx.subtest('replace with an editor keeps tags from other editors and tags without an editor', () => {
+    const updated = forager.media.update(id, {}, ['my:tag'], {editor: 'foo'})
+    ctx.assert.list_partial(updated.tags, [
+      {group: '', name: 'bar_tag'},
+      {group: '', name: 'no_editor_tag'},
+      {group: 'my', name: 'tag'},
+    ], by_slug)
+    ctx.assert.equals(updated.edit_log![0].changes.tags, {added: ['my:tag'], removed: []})
+  })
+
+  await ctx.subtest('replace with an editor replaces tags owned by that editor', () => {
+    const updated = forager.media.update(id, {}, ['my:other_tag'], {editor: 'foo', overwrite: false})
+    ctx.assert.list_partial(updated.tags, [
+      {group: '', name: 'bar_tag'},
+      {group: '', name: 'no_editor_tag'},
+      {group: 'my', name: 'other_tag'},
+    ], by_slug)
+    ctx.assert.equals(updated.edit_log![0].changes.tags, {added: ['my:other_tag'], removed: ['my:tag']})
+  })
+
+  await ctx.subtest('an editor does not overwrite media info fields owned by other editors by default', () => {
+    const updated = forager.media.update(id, {title: 'foo title', description: 'foo description'}, undefined, {editor: 'foo'})
+    ctx.assert.equals(updated.media_reference.title, 'bar title')
+    ctx.assert.equals(updated.media_reference.description, 'foo description')
+  })
+
+  await ctx.subtest('overwrite replaces tags and media info from all editors', () => {
+    const updated = forager.media.update(id, {title: 'foo title'}, ['my:tag'], {editor: 'foo', overwrite: true})
+    ctx.assert.equals(updated.media_reference.title, 'foo title')
+    ctx.assert.list_partial(updated.tags, [{group: 'my', name: 'tag'}])
+  })
+
+  await ctx.subtest('replace without an editor replaces all tags', () => {
+    forager.media.update(id, {}, {add: ['bar_tag']}, {editor: 'bar'})
+    const updated = forager.media.update(id, {}, ['no_editor_tag'])
+    ctx.assert.list_partial(updated.tags, [{group: '', name: 'no_editor_tag'}])
+  })
+})

packages/core/test/ingest.test.ts

@@ -1,6 +1,6 @@
 import { test } from 'forager-test'
 import * as path from '@std/path'
-import { Forager, type MediaFileResponse } from '~/mod.ts'
+import { Forager, FileSystemReceiver, PluginScript, type FileSystemReceiverContext, type ReceiverEditing, type MediaFileResponse } from '~/mod.ts'
 
 test('ingest actions', async ctx => {
   using forager = new Forager(ctx.get_test_config())
@@ -38,6 +38,128 @@ test('ingest actions', async ctx => {
 })
 
 
+class TifReceiver extends FileSystemReceiver {
+  override name = 'tif_scraper'
+  override extensions = ['tif']
+  tags: string[] = []
+  editing?: ReceiverEditing
+  override foreach = async (ctx: FileSystemReceiverContext) => {
+    await this.add(ctx, ctx.entry.path, {title: 'scraped'}, this.tags, undefined, this.editing)
+  }
+}
+
+test('ingest editing', async ctx => {
+  const receiver = new TifReceiver()
+  const plugin_script = new PluginScript()
+  plugin_script.recievers.unshift(receiver)
+  using forager = new Forager(ctx.get_test_config(), plugin_script)
+  forager.init()
+
+  const filepath = ctx.resources.media_files['koch.tif']
+  const tif_glob = ctx.resources.resources_directory + path.SEPARATOR + '*.tif'
+  const by_name = (a: {name: string}, b: {name: string}) => a.name.localeCompare(b.name)
+
+  const manual_media = await forager.media.create(filepath, {}, ['manual_tag'], {editor: 'manual'})
+  const media_reference_id = manual_media.media_reference.id
+
+  await ctx.subtest('ingest over existing media preserves tags from other editors', async () => {
+    receiver.tags = ['scraper_1']
+    receiver.editing = {editor: 'tif_scraper', overwrite: false}
+    await forager.filesystem.discover({path: tif_glob})
+    const ingest = await forager.ingest.start()
+    ctx.assert.equals(ingest.stats.updated, 1)
+
+    const media = forager.media.get({media_reference_id}) as MediaFileResponse
+    ctx.assert.equals(media.media_reference.title, 'scraped')
+    ctx.assert.list_partial(media.tags, [{name: 'manual_tag'}, {name: 'scraper_1'}], by_name)
+    ctx.assert.equals(media.edit_log![0].editor, 'tif_scraper')
+    ctx.assert.equals(media.edit_log![0].changes.tags, {added: ['scraper_1'], removed: []})
+  })
+
+  await ctx.subtest('reingest only replaces tags owned by the receiver', async () => {
+    receiver.tags = ['scraper_2']
+    await forager.filesystem.discover({path: tif_glob, reingest: true})
+    await forager.ingest.start()
+
+    const media = forager.media.get({media_reference_id}) as MediaFileResponse
+    ctx.assert.list_partial(media.tags, [{name: 'manual_tag'}, {name: 'scraper_2'}], by_name)
+    ctx.assert.equals(media.edit_log![0].editor, 'tif_scraper')
+    ctx.assert.equals(media.edit_log![0].changes.tags, {added: ['scraper_2'], removed: ['scraper_1']})
+  })
+
+  await ctx.subtest('a different editor keeps tags from the previous receiver editor', async () => {
+    receiver.tags = ['custom_1']
+    receiver.editing = {editor: 'custom_editor', overwrite: false}
+    await forager.filesystem.discover({path: tif_glob, reingest: true})
+    await forager.ingest.start()
+
+    const media = forager.media.get({media_reference_id}) as MediaFileResponse
+    ctx.assert.list_partial(media.tags, [{name: 'custom_1'}, {name: 'manual_tag'}, {name: 'scraper_2'}], by_name)
+    ctx.assert.equals(media.edit_log![0].editor, 'custom_editor')
+  })
+
+  await ctx.subtest('receiver can overwrite edits by others', async () => {
+    receiver.tags = ['overwrite_1']
+    receiver.editing = {editor: 'tif_scraper', overwrite: true}
+    await forager.filesystem.discover({path: tif_glob, reingest: true})
+    await forager.ingest.start()
+
+    const media = forager.media.get({media_reference_id}) as MediaFileResponse
+    ctx.assert.list_partial(media.tags, [{name: 'overwrite_1'}])
+    ctx.assert.equals(media.edit_log![0].editor, 'tif_scraper')
+  })
+
+  await ctx.subtest('receiver without editing options does not record an editor', async () => {
+    receiver.tags = ['no_editor_1']
+    receiver.editing = undefined
+    const edit_log_count = (forager.media.get({media_reference_id}) as MediaFileResponse).edit_log!.length
+    await forager.filesystem.discover({path: tif_glob, reingest: true})
+    await forager.ingest.start()
+
+    const media = forager.media.get({media_reference_id}) as MediaFileResponse
+    ctx.assert.list_partial(media.tags, [{name: 'no_editor_1'}])
+    ctx.assert.equals(media.edit_log!.length, edit_log_count)
+  })
+})
+
+test('ingest editing on created media', async ctx => {
+  const receiver = new TifReceiver()
+  receiver.tags = ['created_tag']
+  receiver.editing = {editor: 'custom_editor'}
+  const plugin_script = new PluginScript()
+  plugin_script.recievers.unshift(receiver)
+  using forager = new Forager(ctx.get_test_config(), plugin_script)
+  forager.init()
+
+  await forager.filesystem.discover({path: ctx.resources.resources_directory + path.SEPARATOR + '*.tif'})
+  const ingest = await forager.ingest.start()
+  ctx.assert.equals(ingest.stats.created, 1)
+
+  const media = forager.media.get({filepath: ctx.resources.media_files['koch.tif']}) as MediaFileResponse
+  ctx.assert.equals(media.media_reference.editors, ['custom_editor'])
+  ctx.assert.list_partial(media.tags, [{name: 'created_tag'}])
+  ctx.assert.list_partial(media.edit_log!, [{editor: 'custom_editor', operation_type: 'CREATE'}])
+})
+
+test('ingest without editing on created media', async ctx => {
+  const receiver = new TifReceiver()
+  receiver.tags = ['created_tag']
+  const plugin_script = new PluginScript()
+  plugin_script.recievers.unshift(receiver)
+  using forager = new Forager(ctx.get_test_config(), plugin_script)
+  forager.init()
+
+  await forager.filesystem.discover({path: ctx.resources.resources_directory + path.SEPARATOR + '*.tif'})
+  const ingest = await forager.ingest.start()
+  ctx.assert.equals(ingest.stats.created, 1)
+
+  const media = forager.media.get({filepath: ctx.resources.media_files['koch.tif']}) as MediaFileResponse
+  ctx.assert.equals(media.media_reference.editors, null)
+  ctx.assert.list_partial(media.tags, [{name: 'created_tag'}])
+  ctx.assert.equals(media.edit_log!.length, 0)
+})
+
+
 /*
 test('filesystem discovery', async (ctx) => {
   using forager = new Forager(ctx.get_test_config())