From e1783a5238c07ee2361e8e0a55002c03ab28f8d5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 13 Oct 2026 22:02:51 +0000 Subject: [PATCH 1/2] impl(core): media_filepath model and media_file md5checksum media files can now have multiple filepaths. creating media with a duplicate checksum attaches the new filepath to the existing media file and applies its media info and tags as an update Co-authored-by: kaisea.rpi --- packages/core/src/actions/ingest_actions.ts | 13 +- packages/core/src/actions/lib/base.ts | 193 +++++++++++++----- packages/core/src/actions/media_actions.ts | 1 + .../core/src/db/migrations/migration_v12.ts | 40 ++++ packages/core/src/db/migrations/mod.ts | 1 + .../core/src/db/migrations/seed_migration.ts | 20 +- packages/core/src/lib/file_processor.ts | 17 +- packages/core/src/models/lib/result_types.ts | 1 + packages/core/src/models/media_file.ts | 16 +- packages/core/src/models/media_filepath.ts | 64 ++++++ packages/core/src/models/media_reference.ts | 2 +- packages/core/src/models/mod.ts | 1 + packages/core/test/migrations.test.ts | 7 +- 13 files changed, 305 insertions(+), 71 deletions(-) create mode 100644 packages/core/src/db/migrations/migration_v12.ts create mode 100644 packages/core/src/models/media_filepath.ts diff --git a/packages/core/src/actions/ingest_actions.ts b/packages/core/src/actions/ingest_actions.ts index d3fc5301..391b83e8 100644 --- a/packages/core/src/actions/ingest_actions.ts +++ b/packages/core/src/actions/ingest_actions.ts @@ -156,18 +156,19 @@ class IngestActions extends Actions { } try { - const created_media = await ctx.forager.media.create(filepath, media_info, tags, {editor: ctx.receiver.name}) - return {media_reference_id: created_media.media_reference.id, checksum: created_media.media_file.checksum, status: 'created'} + const { status, media } = await this.media_create_with_status(filepath, media_info, tags, {editor: ctx.receiver.name}) + return { + media_reference_id: media.media_reference.id, + checksum: media.media_file.checksum, + status: status === 'created' ? 'created' : 'duplicate', + } } catch (e) { let file_identifier = filepath if (filepath !== ctx.entry.path) { file_identifier = `${filepath} from ${ctx.entry.path}` } - if (e instanceof errors.DuplicateMediaError) { - ctx.logger.warn(`${file_identifier} has a duplicate checksum (${e.checksum}) to ${e.existing_media_filepath}, skipping`) - return {media_reference_id: e.media_reference_id, checksum: e.checksum, status: 'duplicate'} - } else if (e instanceof errors.AlreadyExistsError) { + if (e instanceof errors.AlreadyExistsError) { if (!media_info || !tags) { ctx.logger.info(`${e.identifier} already exists in database, skipping`) return {media_reference_id: e.media_reference_id, status: 'existing'} diff --git a/packages/core/src/actions/lib/base.ts b/packages/core/src/actions/lib/base.ts index 3b20bc53..44eebf0b 100644 --- a/packages/core/src/actions/lib/base.ts +++ b/packages/core/src/actions/lib/base.ts @@ -30,8 +30,17 @@ export interface MediaFileResponse { tags: result_types.Tag[] thumbnails: result_types.PaginatedResult edit_log?: result_types.EditLog[] + /** Every known filepath of the media file. The first filepath found is also stored on `media_file.filepath` */ + filepaths?: result_types.MediaFilepath[] } +/** + * - `created`: a new media file was created + * - `linked`: the file is a duplicate of an existing media file, so its filepath was added to the existing media file + * - `moved`: the file is a duplicate of an existing media file whose filepath no longer exists, so the existing media file was moved to the new filepath + */ +type MediaCreateStatus = 'created' | 'linked' | 'moved' + /** * Structure returned from {@linkcode MediaActions.prototype.search}. Can be either a media series or an individual media file. */ @@ -99,6 +108,14 @@ abstract class Actions extends Emitter { + const { media } = await this.media_create_with_status(filepath, media_info, tags, editing) + return media + } + + /** + * Same as {@linkcode Actions.prototype.media_create}, but also reports whether a new media file was created, or the filepath was attached to an existing media file with the same checksum. + */ + protected async media_create_with_status(filepath: string, media_info?: inputs.MediaInfo, tags?: inputs.MediaReferenceUpdateTags, editing?: CreateEditor): Promise<{status: MediaCreateStatus; media: MediaFileResponse}> { const start_time = performance.now() const parsed = { filepath: parsers.Filepath.parse(filepath), @@ -109,32 +126,13 @@ abstract class Actions extends Emitter extends Emitter extends Emitter { + const { media_reference_id } = existing_media_file + const existing_media_filepath = this.models.MediaFilepath.select_one({filepath: params.filepath}) + if (existing_media_filepath) { + if (existing_media_filepath.media_file_id !== existing_media_file.id) { + const media_file_at_filepath = this.models.MediaFile.select_one({filepath: params.filepath}, {or_raise: true}) + throw new errors.ChecksumMismatchError(params.filepath, media_file_at_filepath.checksum, params.checksum) + } + throw new errors.MediaAlreadyExistsError(params.filepath, params.checksum, media_reference_id) + } + + const primary_file_exists = await fs.exists(existing_media_file.filepath) + + // TODO media reference tags do not track which filepath they came from, so tags from a duplicate filepath can only be added, never replaced. + // In the future we may want media_reference_tag to store its filepath source so we can merge tags better. + const tags: outputs.MediaReferenceUpdateTags = { + add: [...(params.tags.replace ?? []), ...params.tags.add], + remove: [], + } + const has_updates = Object.keys(params.media_info).length > 0 || tags.add.length > 0 + + const transaction = this.ctx.db.transaction_sync(() => { + if (primary_file_exists) { + this.ctx.logger.info(`File ${params.filepath} has a duplicate checksum in the database at ${existing_media_file.filepath}. Adding it as another filepath of the existing media file.`) + this.models.MediaFilepath.create({ + media_file_id: existing_media_file.id, + filepath: params.filepath, + filename: params.filename, + }) + } else { + // TODO we may want to move this behind a flag for filesystem ingest, or a config flag + // I worry that we may end up replacing files on the fly when we don't want to + this.ctx.logger.warn(`File ${params.filepath} has a duplicate checksum in the database at ${existing_media_file.filepath}. This existing file is missing, so the path is being updated to the new one.`) + const primary_media_filepath = this.models.MediaFilepath.select_one({filepath: existing_media_file.filepath}, {or_raise: true}) + this.models.MediaFilepath.update_filepath({id: primary_media_filepath.id, filepath: params.filepath, filename: params.filename}) + this.models.MediaFile.update_filepath({id: existing_media_file.id, filepath: params.filepath, filename: params.filename}) + } + + if (existing_media_file.md5checksum === null) { + this.models.MediaFile.update_md5checksum({id: existing_media_file.id, md5checksum: params.md5checksum}) + } + + if (has_updates) { + this.#media_update_records(media_reference_id, params.media_info, tags, params.editing) + } + }) + transaction() + + return { + status: primary_file_exists ? 'linked' : 'moved', + media: this.get_media_file_result({ + media_reference_id, + media_file_id: existing_media_file.id, + thumbnail_limit: 1, + }), + } } protected media_update(media_reference_id: number, media_info?: inputs.MediaInfo, tags?: inputs.MediaReferenceUpdateTags, editing?: UpdateEditor) { @@ -196,38 +271,7 @@ abstract class Actions extends Emitter { - const media_info_updates = {...parsed.media_info} - - // when overwrite is false, we will only overwrite fields that are already owned by this editor (or by no editors at all) - if (parsed.editing?.overwrite === false) { - const media_info_last_editors = this.models.EditLog.get_media_info_last_editors({media_reference_id}) - const media_info_update_fields = Object.keys(media_info_updates) as (keyof outputs.MediaInfo)[] - for (const media_info_field of media_info_update_fields) { - if (media_info_last_editors[media_info_field] && media_info_last_editors[media_info_field] !== parsed.editing.editor) { - delete media_info_updates[media_info_field] - } - } - - } - - this.models.MediaReference.update({ - id: media_reference_id, - ...media_info_updates, - }) - - const tag_changes = this.manage_media_tags(media_reference_id, parsed.tags, parsed.editing) - - if (parsed.editing?.editor) { - this.models.EditLog.create({ - media_reference_id: media_reference_id, - operation_type: 'UPDATE', - editor: parsed.editing?.editor, - changes: { - media_info: media_info_updates, - tags: tag_changes, - } - }) - } + this.#media_update_records(media_reference_id, parsed.media_info, parsed.tags, parsed.editing) }) transaction() @@ -241,6 +285,42 @@ abstract class Actions extends Emitter extends Emitter extends Emitter { + console.log(`Adding md5checksum column to media_file table`) + this.driver.exec(`ALTER TABLE media_file ADD COLUMN md5checksum TEXT`) + this.driver.exec(`CREATE INDEX media_file_md5checksum ON media_file (md5checksum)`) + + // sqlite indexes and tables share a namespace, so the old media_file filepath index needs a new name + this.driver.exec(`DROP INDEX media_filepath`) + this.driver.exec(`CREATE UNIQUE INDEX media_file_filepath ON media_file (filepath)`) + + console.log(`Creating media_filepath table`) + this.driver.exec(` + CREATE TABLE media_filepath ( + id INTEGER PRIMARY KEY NOT NULL, + media_file_id INTEGER NOT NULL, + filepath TEXT NOT NULL, + filename TEXT NOT NULL, + updated_at ${TIMESTAMP_COLUMN}, + created_at ${TIMESTAMP_COLUMN}, + + FOREIGN KEY (media_file_id) REFERENCES media_file(id) + ) + `) + this.driver.exec(` + INSERT INTO media_filepath (media_file_id, filepath, filename, updated_at, created_at) + SELECT id, filepath, filename, created_at, created_at + FROM media_file + ORDER BY id + `) + this.driver.exec(`CREATE UNIQUE INDEX media_filepath_lookup ON media_filepath (filepath)`) + this.driver.exec(`CREATE INDEX media_filepath_media_file ON media_filepath (media_file_id)`) + } +} diff --git a/packages/core/src/db/migrations/mod.ts b/packages/core/src/db/migrations/mod.ts index abf9f68f..0dacd292 100644 --- a/packages/core/src/db/migrations/mod.ts +++ b/packages/core/src/db/migrations/mod.ts @@ -9,5 +9,6 @@ import './migration_v8.ts' import './migration_v9.ts' import './migration_v10.ts' import './migration_v11.ts' +import './migration_v12.ts' export { migrations } from './registry.ts' diff --git a/packages/core/src/db/migrations/seed_migration.ts b/packages/core/src/db/migrations/seed_migration.ts index 30e0d3de..b398d84a 100644 --- a/packages/core/src/db/migrations/seed_migration.ts +++ b/packages/core/src/db/migrations/seed_migration.ts @@ -4,7 +4,7 @@ import { migrations, sql, TIMESTAMP_SQLITE, TIMESTAMP_COLUMN, TIMESTAMP_COLUMN_O @migrations.register() export class Migration extends torm.SeedMigration { - version = 11 + version = 12 sql = sql` CREATE TABLE media_file ( @@ -15,6 +15,7 @@ export class Migration extends torm.SeedMigration { -- mime_type TEXT NOT NULL, file_size_bytes INTEGER NOT NULL, checksum TEXT NOT NULL UNIQUE, + md5checksum TEXT, -- image,video,audio media_type TEXT NOT NULL CHECK( media_type IN ('IMAGE', 'VIDEO', 'AUDIO') ), @@ -37,6 +38,18 @@ export class Migration extends torm.SeedMigration { FOREIGN KEY (media_reference_id) REFERENCES media_reference(id) ); + -- every known location of a media file. media_file.filepath is a denormalized reference to the first filepath found + CREATE TABLE media_filepath ( + id INTEGER PRIMARY KEY NOT NULL, + media_file_id INTEGER NOT NULL, + filepath TEXT NOT NULL, + filename TEXT NOT NULL, + updated_at ${TIMESTAMP_COLUMN}, + created_at ${TIMESTAMP_COLUMN}, + + FOREIGN KEY (media_file_id) REFERENCES media_file(id) + ); + CREATE TABLE media_thumbnail ( id INTEGER PRIMARY KEY NOT NULL, media_timestamp FLOAT NOT NULL, @@ -291,7 +304,10 @@ export class Migration extends torm.SeedMigration { CREATE INDEX tag_parent_child_slug ON tag_parent (child_tag_slug); CREATE INDEX tag_parent_slug ON tag_parent (parent_tag_slug); CREATE UNIQUE INDEX media_file_reference ON media_file (media_reference_id); - CREATE UNIQUE INDEX media_filepath ON media_file (filepath); + CREATE UNIQUE INDEX media_file_filepath ON media_file (filepath); + CREATE INDEX media_file_md5checksum ON media_file (md5checksum); + CREATE UNIQUE INDEX media_filepath_lookup ON media_filepath (filepath); + CREATE INDEX media_filepath_media_file ON media_filepath (media_file_id); CREATE INDEX media_reference_edit_log ON edit_log (media_reference_id, created_at); CREATE UNIQUE INDEX filesystem_path_lookup ON filesystem_path (filepath); CREATE UNIQUE INDEX filesystem_path_priority ON filesystem_path (ingest_priority) WHERE directory = 0; diff --git a/packages/core/src/lib/file_processor.ts b/packages/core/src/lib/file_processor.ts index a1471518..f75bb0f1 100644 --- a/packages/core/src/lib/file_processor.ts +++ b/packages/core/src/lib/file_processor.ts @@ -383,12 +383,23 @@ class FileProcessor { } public async get_checksum() { - const hash = node_crypto.createHash('sha256') + const { checksum } = await this.get_checksums() + return checksum + } + + /** computes the sha256 checksum (forager's primary identifier for a file) and the md5 checksum in a single read of the file */ + public async get_checksums() { + const sha256_hash = node_crypto.createHash('sha256') + const md5_hash = node_crypto.createHash('md5') using file = await Deno.open(this.#filepath) for await (const chunk of file.readable) { - hash.update(chunk) + sha256_hash.update(chunk) + md5_hash.update(chunk) + } + return { + checksum: sha256_hash.digest('hex'), + md5checksum: md5_hash.digest('hex'), } - return hash.digest('hex') } public async get_size() { diff --git a/packages/core/src/models/lib/result_types.ts b/packages/core/src/models/lib/result_types.ts index e79e8bf2..f8927824 100644 --- a/packages/core/src/models/lib/result_types.ts +++ b/packages/core/src/models/lib/result_types.ts @@ -5,6 +5,7 @@ export type MediaReference = InferSchemaTypes export type MediaSeriesItem = InferSchemaTypes export type MediaFile = InferSchemaTypes +export type MediaFilepath = InferSchemaTypes export type MediaThumbnail = InferSchemaTypes export type TagGroup = InferSchemaTypes export type Tag = InferSchemaTypes & {group: TagGroup['name']; color: TagGroup['color']} diff --git a/packages/core/src/models/media_file.ts b/packages/core/src/models/media_file.ts index 361854b9..fbd9a1e3 100644 --- a/packages/core/src/models/media_file.ts +++ b/packages/core/src/models/media_file.ts @@ -10,6 +10,7 @@ class MediaFile extends Model { thumbnail_directory_path: field.string(), file_size_bytes: field.number(), checksum: field.string(), + md5checksum: field.string().optional(), media_type: field.string(), // TODO enum support codec: field.string(), content_type: field.string(), @@ -32,7 +33,8 @@ class MediaFile extends Model { #select_by_filepath = this.query` SELECT ${MediaFile.result['*']} FROM media_file - WHERE filepath = ${MediaFile.params.filepath}` + INNER JOIN media_filepath ON media_filepath.media_file_id = media_file.id + WHERE media_filepath.filepath = ${MediaFile.params.filepath}` #select_by_checksum = this.query` SELECT ${MediaFile.result['*']} FROM media_file @@ -44,6 +46,7 @@ class MediaFile extends Model { thumbnail_directory_path, file_size_bytes, checksum, + md5checksum, media_type, content_type, codec, @@ -61,6 +64,7 @@ class MediaFile extends Model { MediaFile.params.thumbnail_directory_path, MediaFile.params.file_size_bytes, MediaFile.params.checksum, + MediaFile.params.md5checksum, MediaFile.params.media_type, MediaFile.params.content_type, MediaFile.params.codec, @@ -76,7 +80,13 @@ class MediaFile extends Model { #update_filepath_by_id = this.query.exec` UPDATE media_file SET - filepath = ${MediaFile.params.filepath} + filepath = ${MediaFile.params.filepath}, + filename = ${MediaFile.params.filename} + WHERE id = ${MediaFile.params.id}` + + #update_md5checksum_by_id = this.query.exec` + UPDATE media_file SET + md5checksum = ${MediaFile.params.md5checksum} WHERE id = ${MediaFile.params.id}` #delete_by_id = this.query.exec` @@ -109,6 +119,8 @@ class MediaFile extends Model { public update_filepath = this.#update_filepath_by_id + public update_md5checksum = this.#update_md5checksum_by_id + public delete = this.delete_fn(this.#delete_by_id) } diff --git a/packages/core/src/models/media_filepath.ts b/packages/core/src/models/media_filepath.ts new file mode 100644 index 00000000..35c3501e --- /dev/null +++ b/packages/core/src/models/media_filepath.ts @@ -0,0 +1,64 @@ +import * as torm from '@torm/sqlite' +import { Model, field } from '~/models/lib/base.ts' +import { TIMESTAMP_SQLITE } from '~/db/migrations/registry.ts' + +class MediaFilepath extends Model { + static schema = torm.schema('media_filepath', { + id: field.number(), + media_file_id: field.number(), + filepath: field.string(), + filename: field.string(), + updated_at: field.datetime(), + created_at: field.datetime(), + }) + static params = this.schema.params + static result = this.schema.result + + #create = this.query.one`INSERT INTO media_filepath ( + media_file_id, + filepath, + filename + ) VALUES (${[ + MediaFilepath.params.media_file_id, + MediaFilepath.params.filepath, + MediaFilepath.params.filename, + ]}) RETURNING ${MediaFilepath.result.id}` + + #select_by_filepath = this.query` + SELECT ${MediaFilepath.result['*']} FROM media_filepath + WHERE filepath = ${MediaFilepath.params.filepath}` + + #select_many_by_media_file_id = this.query` + SELECT ${MediaFilepath.result['*']} FROM media_filepath + WHERE media_file_id = ${MediaFilepath.params.media_file_id} + ORDER BY id` + + #update_filepath_by_id = this.query.exec` + UPDATE media_filepath SET + filepath = ${MediaFilepath.params.filepath}, + filename = ${MediaFilepath.params.filename}, + updated_at = ${TIMESTAMP_SQLITE} + WHERE id = ${MediaFilepath.params.id}` + + #delete_by_media_file_id = this.query.exec` + DELETE FROM media_filepath + WHERE media_file_id = ${MediaFilepath.params.media_file_id}` + + #select_one_impl(params: { filepath: string }) { + return this.#select_by_filepath.one(params) + } + + public select_one = this.select_one_fn(this.#select_one_impl.bind(this)) + + public select_many(params: { media_file_id: number }) { + return this.#select_many_by_media_file_id.all(params) + } + + public create = this.create_fn(this.#create) + + public update_filepath = this.#update_filepath_by_id + + public delete = this.delete_fn(this.#delete_by_media_file_id) +} + +export { MediaFilepath } diff --git a/packages/core/src/models/media_reference.ts b/packages/core/src/models/media_reference.ts index 10ed5017..ba896a03 100644 --- a/packages/core/src/models/media_reference.ts +++ b/packages/core/src/models/media_reference.ts @@ -465,7 +465,7 @@ ${group_builder.generate_sql()} } if (params.filepath) { - builder.add_where_clause(`media_file.filepath GLOB '${params.filepath}'`) + builder.add_where_clause(`EXISTS (SELECT 1 FROM media_filepath WHERE media_filepath.media_file_id = media_file.id AND media_filepath.filepath GLOB '${params.filepath}')`) } if (params.duration_min !== undefined) { diff --git a/packages/core/src/models/mod.ts b/packages/core/src/models/mod.ts index 4586db0f..653a03a4 100644 --- a/packages/core/src/models/mod.ts +++ b/packages/core/src/models/mod.ts @@ -1,4 +1,5 @@ export {MediaFile} from './media_file.ts' +export {MediaFilepath} from './media_filepath.ts' export {MediaReference} from './media_reference.ts' export {MediaThumbnail} from './media_thumbnail.ts' export {TagGroup} from './tag_group.ts' diff --git a/packages/core/test/migrations.test.ts b/packages/core/test/migrations.test.ts index 028a2535..599e13dd 100644 --- a/packages/core/test/migrations.test.ts +++ b/packages/core/test/migrations.test.ts @@ -3,7 +3,7 @@ import * as fs from '@std/fs' import * as path from '@std/path' import { Forager } from '~/mod.ts' -const CURRENT_VERSION = 11 +const CURRENT_VERSION = 12 test('migrate from v1 schema', async (ctx) => { const forager_v1_path = ctx.create_fixture_path('forager_v1') @@ -72,6 +72,11 @@ test('migrate from v1 schema', async (ctx) => { next_version: 11, backup: true, }, + { + start_version: 11, + next_version: 12, + backup: true, + }, ] ctx.assert.equals(v1_migration_info.db.migration_operations, expected_migration_operations) From aaf0477a6d0d373b213022305aac190c9bf184bf Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 13 Oct 2026 22:05:29 +0000 Subject: [PATCH 2/2] test(core): cover multiple filepaths per media file and resolve duplicate filepaths absolutely Co-authored-by: kaisea.rpi --- packages/core/src/actions/lib/base.ts | 2 +- packages/core/test/media_filepath.test.ts | 113 ++++++++++++++++++++++ packages/core/test/migrations.test.ts | 11 +++ 3 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 packages/core/test/media_filepath.test.ts diff --git a/packages/core/src/actions/lib/base.ts b/packages/core/src/actions/lib/base.ts index 44eebf0b..f3a84503 100644 --- a/packages/core/src/actions/lib/base.ts +++ b/packages/core/src/actions/lib/base.ts @@ -130,7 +130,7 @@ abstract class Actions extends Emitter { + using forager = new Forager(ctx.get_test_config()) + forager.init() + + const originals_folder = path.resolve(ctx.create_fixture_path('originals')) + const copies_folder = path.resolve(ctx.create_fixture_path('copies')) + await Deno.mkdir(originals_folder, { recursive: true }) + await Deno.mkdir(copies_folder, { recursive: true }) + const original_filepath = path.join(originals_folder, 'cat_doodle.jpg') + const copy_filepath = path.join(copies_folder, 'cat_doodle_copy.jpg') + const moved_filepath = path.join(copies_folder, 'cat_doodle_moved.jpg') + await Deno.copyFile(ctx.resources.media_files['cat_doodle.jpg'], original_filepath) + await Deno.copyFile(ctx.resources.media_files['cat_doodle.jpg'], copy_filepath) + await Deno.copyFile(ctx.resources.media_files['cat_doodle.jpg'], moved_filepath) + + const media_original = await forager.media.create(original_filepath, {title: 'Original Doodle'}, ['cat']) + + await ctx.subtest('created media has a single filepath and an md5checksum', () => { + ctx.assert.equals(media_original.media_file.filepath, original_filepath) + ctx.assert.equals(media_original.media_file.md5checksum, CAT_DOODLE_MD5) + ctx.assert.equals(media_original.filepaths?.map(f => f.filepath), [original_filepath]) + }) + + await ctx.subtest('duplicate checksum adds a filepath to the existing media file', async () => { + const media_copy = await forager.media.create(copy_filepath, {description: 'from the copy'}, ['doodle']) + ctx.assert.equals(media_copy.media_reference.id, media_original.media_reference.id) + ctx.assert.equals(media_copy.media_file.id, media_original.media_file.id) + // the media file keeps a reference to the first filepath found + ctx.assert.equals(media_copy.media_file.filepath, original_filepath) + ctx.assert.equals(media_copy.filepaths?.map(f => f.filepath), [original_filepath, copy_filepath]) + ctx.assert.list_partial(media_copy.filepaths!, [{filename: 'cat_doodle.jpg'}, {filename: 'cat_doodle_copy.jpg'}]) + + // media info and tags are applied like an update, and tags are merged rather than replaced + ctx.assert.object_match(media_copy.media_reference, {title: 'Original Doodle', description: 'from the copy'}) + ctx.assert.equals(media_copy.tags.map(t => t.name).sort(), ['cat', 'doodle']) + + ctx.assert.search_result(forager.media.search(), {total: 1}) + }) + + await ctx.subtest('adding the same duplicate filepath again is guarded', async () => { + await ctx.assert.rejects(() => forager.media.create(copy_filepath), errors.MediaAlreadyExistsError) + await ctx.assert.rejects(() => forager.media.create(original_filepath), errors.MediaAlreadyExistsError) + }) + + await ctx.subtest('media can be retrieved and searched by any of its filepaths', () => { + const media_by_copy = forager.media.get({filepath: copy_filepath}) + ctx.assert.equals(media_by_copy.media_reference.id, media_original.media_reference.id) + + ctx.assert.search_result(forager.media.search({query: {filepath: `*cat_doodle_copy.jpg`}}), { + total: 1, + results: [{media_reference: {id: media_original.media_reference.id}}], + }) + ctx.assert.search_result(forager.media.search({query: {filepath: `*cat_doodle.jpg`}}), { + total: 1, + results: [{media_reference: {id: media_original.media_reference.id}}], + }) + }) + + await ctx.subtest('a missing original filepath is swapped for the new filepath', async () => { + await Deno.remove(original_filepath) + const media_moved = await forager.media.create(moved_filepath) + ctx.assert.equals(media_moved.media_reference.id, media_original.media_reference.id) + ctx.assert.equals(media_moved.media_file.filepath, moved_filepath) + ctx.assert.equals(media_moved.media_file.filename, 'cat_doodle_moved.jpg') + ctx.assert.equals(media_moved.filepaths?.map(f => f.filepath), [moved_filepath, copy_filepath]) + ctx.assert.equals(media_moved.tags.map(t => t.name).sort(), ['cat', 'doodle']) + + ctx.assert.throws(() => forager.media.get({filepath: original_filepath}), errors.NotFoundError) + }) + + await ctx.subtest('deleting media removes all of its filepaths', async () => { + await forager.media.delete({filepath: copy_filepath}) + ctx.assert.search_result(forager.media.search(), {total: 0}) + ctx.assert.throws(() => forager.media.get({filepath: copy_filepath}), errors.NotFoundError) + ctx.assert.throws(() => forager.media.get({filepath: moved_filepath}), errors.NotFoundError) + + // the filepaths are free to be used again + const media_recreated = await forager.media.create(copy_filepath) + ctx.assert.equals(media_recreated.filepaths?.map(f => f.filepath), [copy_filepath]) + }) +}) + +test('ingest duplicate filepaths', async ctx => { + using forager = new Forager(ctx.get_test_config()) + forager.init() + + const ingest_folder = path.resolve(ctx.create_fixture_path('ingest')) + await Deno.mkdir(path.join(ingest_folder, 'a'), { recursive: true }) + await Deno.mkdir(path.join(ingest_folder, 'b'), { recursive: true }) + const filepath_a = path.join(ingest_folder, 'a', 'cat_doodle.jpg') + const filepath_b = path.join(ingest_folder, 'b', 'cat_doodle.jpg') + await Deno.copyFile(ctx.resources.media_files['cat_doodle.jpg'], filepath_a) + await Deno.copyFile(ctx.resources.media_files['cat_doodle.jpg'], filepath_b) + + await forager.filesystem.discover({path: ingest_folder + path.SEPARATOR + '**/*.jpg'}) + const ingest = await forager.ingest.start({set: {tags: [{name: 'doodle'}]}}) + ctx.assert.equals(ingest.stats, {created: 1, updated: 0, existing: 0, duplicate: 1, errored: 0}) + + ctx.assert.search_result(forager.media.search(), {total: 1}) + const media_a = forager.media.get({filepath: filepath_a}) + const media_b = forager.media.get({filepath: filepath_b}) + ctx.assert.equals(media_a.media_reference.id, media_b.media_reference.id) + if (media_a.media_type !== 'media_file') throw new Error('expected a media file') + ctx.assert.equals(media_a.media_file.md5checksum, CAT_DOODLE_MD5) + ctx.assert.equals(media_a.filepaths?.map(f => f.filepath).sort(), [filepath_a, filepath_b]) + ctx.assert.equals(media_a.tags.map(t => t.name), ['doodle']) +}, {skip: {os: 'windows'}}) diff --git a/packages/core/test/migrations.test.ts b/packages/core/test/migrations.test.ts index 599e13dd..e1b7460d 100644 --- a/packages/core/test/migrations.test.ts +++ b/packages/core/test/migrations.test.ts @@ -93,6 +93,17 @@ test('migrate from v1 schema', async (ctx) => { ] }) + await ctx.subtest('existing media files are migrated to media filepaths', () => { + for (const result of forager.media.search().results) { + if (result.media_type !== 'media_file') throw new Error('expected a media file') + const media = forager.media.get({filepath: result.media_file.filepath}) + if (media.media_type !== 'media_file') throw new Error('expected a media file') + ctx.assert.equals(media.media_reference.id, result.media_reference.id) + ctx.assert.equals(media.media_file.md5checksum, null) + ctx.assert.list_partial(media.filepaths!, [{filepath: result.media_file.filepath, filename: result.media_file.filename}]) + } + }) + // check if the schemas are identical between a freshly seeded database, and a database migrated from v1 const forager_new = new Forager({ database: {folder: forager_new_path, backups: true},