|
| 1 | +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. |
| 2 | + |
| 3 | +/** |
| 4 | + * #26533 — the pending-upload retention never reaps a file a record holds. |
| 5 | + * |
| 6 | + * `sys_file` declares a 7-day retention on `status: pending` rows: a presigned |
| 7 | + * upload nobody completed is reclaimed, row and bytes, by the platform sweep |
| 8 | + * through {@link createSysFileReapGuard}. Before #26531 a signed-in record |
| 9 | + * write could claim such a file: the record kept the id, and the row was |
| 10 | + * stamped with `ref_*` while it stayed `pending`. A system write still leaves |
| 11 | + * that state today (the seed loader writes the record before `claimFiles` |
| 12 | + * commits the row). The guard's pending branch deleted the bytes without |
| 13 | + * asking whether anything held the file, so seven days later the record named |
| 14 | + * an id whose bytes were gone. |
| 15 | + * |
| 16 | + * Now the pending branch asks the one hold question the `deleted` branch asks |
| 17 | + * ({@link findFileHolder}): a held pending file is kept and named in a warning |
| 18 | + * for an operator, and stays `pending` (committing it would skip the byte |
| 19 | + * check `/upload/complete` makes, #24548). An unheld pending file of the same |
| 20 | + * age is reaped as before. |
| 21 | + * |
| 22 | + * Driven as it ships: one real `ObjectQL` engine over sqlite `:memory:`, |
| 23 | + * `StorageServicePlugin` booted on the local adapter (its own reap guard, |
| 24 | + * registered with a real `LifecycleService`), the upload doors called as |
| 25 | + * routes, and the sweep run with its clock past the retention. |
| 26 | + */ |
| 27 | + |
| 28 | +import { describe, it, expect, vi, beforeAll, afterAll } from 'vitest'; |
| 29 | +import { promises as fs } from 'node:fs'; |
| 30 | +import { join } from 'node:path'; |
| 31 | +import { tmpdir } from 'node:os'; |
| 32 | +import type { RouteHandler } from '@objectstack/spec/contracts'; |
| 33 | +import type { IStorageService } from '@objectstack/spec/contracts'; |
| 34 | +import { RAW_FILE_VALUES_CONTEXT_KEY, type EngineQueryOptions } from '@objectstack/spec/data'; |
| 35 | +import { ObjectQL, LifecycleService } from '@objectstack/objectql'; |
| 36 | +import { SqlDriver } from '@objectstack/driver-sql'; |
| 37 | +import { StorageServicePlugin } from './storage-service-plugin.js'; |
| 38 | +import { installDefaultOrganization } from './default-organization.testkit.js'; |
| 39 | + |
| 40 | +const BASE = '/api/v1/storage'; |
| 41 | +const SYS = { isSystem: true }; |
| 42 | +const SYS_RAW = { isSystem: true, [RAW_FILE_VALUES_CONTEXT_KEY]: true }; |
| 43 | +const WRITER = 'u_writer'; |
| 44 | +const DAY_MS = 24 * 60 * 60 * 1000; |
| 45 | + |
| 46 | +const recordingLogger = () => ({ |
| 47 | + info: vi.fn(), warn: vi.fn(), debug: vi.fn(), error: vi.fn(), |
| 48 | + trace: vi.fn(), fatal: vi.fn(), child() { return this; }, |
| 49 | +}); |
| 50 | + |
| 51 | +type Res = { _status: number; _json: any }; |
| 52 | + |
| 53 | +describe('#26533 — the pending-upload retention never reaps a file a record holds', () => { |
| 54 | + let rootDir: string; |
| 55 | + let engine: ObjectQL; |
| 56 | + let lifecycle: LifecycleService; |
| 57 | + let storage: IStorageService; |
| 58 | + const logger = recordingLogger(); |
| 59 | + const routes = new Map<string, RouteHandler>(); |
| 60 | + |
| 61 | + beforeAll(async () => { |
| 62 | + rootDir = join(tmpdir(), `os-26533-${Date.now()}-${Math.random().toString(36).slice(2)}`); |
| 63 | + await fs.mkdir(rootDir, { recursive: true }); |
| 64 | + |
| 65 | + engine = new ObjectQL({ logger: recordingLogger() } as any); |
| 66 | + engine.registerDriver(new SqlDriver({ |
| 67 | + client: 'better-sqlite3', |
| 68 | + connection: { filename: ':memory:' }, |
| 69 | + useNullAsDefault: true, |
| 70 | + }) as any, true); |
| 71 | + await engine.init(); |
| 72 | + |
| 73 | + // The real engine, except that a read of an object this fixture never |
| 74 | + // registered — the permission store the door's context resolution |
| 75 | + // consults — answers empty. |
| 76 | + const dataEngine = new Proxy(engine, { |
| 77 | + get(target, prop) { |
| 78 | + if (prop === 'find') { |
| 79 | + return (object: string, query?: EngineQueryOptions) => |
| 80 | + target.getSchema(object) ? target.find(object, query) : Promise.resolve([]); |
| 81 | + } |
| 82 | + const value = Reflect.get(target, prop, target); |
| 83 | + return typeof value === 'function' ? value.bind(target) : value; |
| 84 | + }, |
| 85 | + }); |
| 86 | + |
| 87 | + // The sweep's clock stands eight days ahead: every row created below is |
| 88 | + // past the 7-day pending-upload retention. |
| 89 | + lifecycle = new LifecycleService({ |
| 90 | + getEngine: () => engine as any, |
| 91 | + logger: { info: () => {}, warn: () => {}, debug: () => {} }, |
| 92 | + now: () => Date.now() + 8 * DAY_MS, |
| 93 | + referenceAudit: { enabled: false }, |
| 94 | + }); |
| 95 | + |
| 96 | + const services = new Map<string, unknown>(); |
| 97 | + const ready: Array<() => Promise<void> | void> = []; |
| 98 | + const mount = (m: string) => (path: string, handler: RouteHandler) => { routes.set(`${m}:${path}`, handler); }; |
| 99 | + const ctx = { |
| 100 | + logger, |
| 101 | + registerService: (name: string, svc: unknown) => { services.set(name, svc); }, |
| 102 | + getService: <T>(name: string): T => { |
| 103 | + if (!services.has(name)) throw new Error(`service '${name}' not registered`); |
| 104 | + return services.get(name) as T; |
| 105 | + }, |
| 106 | + hook: (event: string, fn: () => Promise<void> | void) => { if (event === 'kernel:ready') ready.push(fn); }, |
| 107 | + }; |
| 108 | + ctx.registerService('objectql', dataEngine); |
| 109 | + ctx.registerService('lifecycle', lifecycle); |
| 110 | + ctx.registerService('http-server', { |
| 111 | + get: mount('GET'), post: mount('POST'), put: mount('PUT'), |
| 112 | + delete: vi.fn(), patch: vi.fn(), use: vi.fn(), |
| 113 | + listen: vi.fn().mockResolvedValue(undefined), |
| 114 | + close: vi.fn().mockResolvedValue(undefined), |
| 115 | + }); |
| 116 | + ctx.registerService('manifest', { register: (manifest: any) => engine.registerApp(manifest) }); |
| 117 | + ctx.registerService('auth', { |
| 118 | + api: { |
| 119 | + getSession: async ({ headers }: { headers: Headers }) => { |
| 120 | + const userId = headers.get('x-test-user'); |
| 121 | + return userId ? { user: { id: userId }, session: {} } : undefined; |
| 122 | + }, |
| 123 | + }, |
| 124 | + }); |
| 125 | + |
| 126 | + const plugin = new StorageServicePlugin({ adapter: 'local', local: { rootDir }, bindToSettings: false }); |
| 127 | + await plugin.init(ctx as any); |
| 128 | + |
| 129 | + engine.registerApp({ |
| 130 | + id: 'com.objectstack.test.pending-claimed-file-reap', |
| 131 | + name: 'Pending claimed file reap', |
| 132 | + version: '1.0.0', |
| 133 | + type: 'plugin', |
| 134 | + scope: 'system', |
| 135 | + objects: [{ |
| 136 | + name: 'media', |
| 137 | + label: 'Media', |
| 138 | + fields: { |
| 139 | + name: { name: 'name', type: 'text' }, |
| 140 | + poster: { name: 'poster', label: 'Poster', type: 'image' }, |
| 141 | + }, |
| 142 | + }], |
| 143 | + } as never); |
| 144 | + await engine.syncSchemas(); |
| 145 | + await installDefaultOrganization(engine); |
| 146 | + await plugin.start(ctx as any); |
| 147 | + for (const fn of ready) await fn(); |
| 148 | + storage = services.get('storage') as IStorageService; |
| 149 | + }); |
| 150 | + |
| 151 | + afterAll(async () => { |
| 152 | + try { await engine?.destroy(); } catch { /* already torn down */ } |
| 153 | + if (rootDir) await fs.rm(rootDir, { recursive: true, force: true }); |
| 154 | + }); |
| 155 | + |
| 156 | + /** Call a storage route as the writer. */ |
| 157 | + const call = async (method: string, path: string, req: Record<string, unknown>): Promise<Res> => { |
| 158 | + const res: any = { |
| 159 | + _status: 200, _json: null, |
| 160 | + json(data: any) { res._json = data; return res; }, |
| 161 | + send() { return res; }, |
| 162 | + status(code: number) { res._status = code; return res; }, |
| 163 | + header() { return res; }, |
| 164 | + }; |
| 165 | + const handler = routes.get(`${method}:${path}`); |
| 166 | + if (!handler) throw new Error(`no route ${method} ${path}`); |
| 167 | + await handler({ params: {}, query: {}, headers: { 'x-test-user': WRITER }, method, path, ...req } as any, res); |
| 168 | + return res; |
| 169 | + }; |
| 170 | + |
| 171 | + /** Presign, then PUT the bytes. `/upload/complete` is never called, so the row stays `pending`. */ |
| 172 | + const pendingUpload = async (filename: string) => { |
| 173 | + const bytes = Buffer.alloc(100, 1); |
| 174 | + const presigned = await call('POST', `${BASE}/upload/presigned`, { |
| 175 | + body: { filename, mimeType: 'image/png', size: bytes.byteLength }, |
| 176 | + }); |
| 177 | + expect(presigned._status, JSON.stringify(presigned._json)).toBe(200); |
| 178 | + const { fileId, uploadUrl } = presigned._json.data as { fileId: string; uploadUrl: string }; |
| 179 | + const token = uploadUrl.slice(uploadUrl.lastIndexOf('/') + 1); |
| 180 | + const put = await call('PUT', `${BASE}/_local/raw/:token`, { params: { token }, rawBody: async () => bytes }); |
| 181 | + expect(put._status, JSON.stringify(put._json)).toBe(200); |
| 182 | + return fileId; |
| 183 | + }; |
| 184 | + |
| 185 | + const fileRow = async (id: string) => |
| 186 | + ((await engine.find('sys_file', { where: { id }, context: SYS })) as Array<Record<string, any>>)[0] ?? null; |
| 187 | + const record = async (id: string) => |
| 188 | + ((await engine.find('media', { where: { id }, context: SYS_RAW })) as Array<Record<string, any>>)[0]; |
| 189 | + |
| 190 | + it('past the retention, a pending file a record holds keeps its bytes and row; an unheld one is reaped', async () => { |
| 191 | + const claimed = await pendingUpload('claimed.png'); |
| 192 | + const abandoned = await pendingUpload('abandoned.png'); |
| 193 | + // The state a signed-in write left before #26531 (and a system write |
| 194 | + // still leaves): the record holds the id, and the row carries `ref_*` |
| 195 | + // while it stays `pending`. |
| 196 | + await engine.insert('media', { id: 'm_1', name: 'm_1', poster: claimed } as any, { context: SYS }); |
| 197 | + const before = await fileRow(claimed); |
| 198 | + expect(before).toMatchObject({ status: 'pending', ref_object: 'media', ref_id: 'm_1', ref_field: 'poster' }); |
| 199 | + const claimedKey = before!.key as string; |
| 200 | + const abandonedKey = (await fileRow(abandoned))!.key as string; |
| 201 | + expect(await storage.exists(claimedKey)).toBe(true); |
| 202 | + expect(await storage.exists(abandonedKey)).toBe(true); |
| 203 | + |
| 204 | + const report = await lifecycle.sweep(); |
| 205 | + |
| 206 | + expect(report.errors, JSON.stringify(report.errors)).toEqual([]); |
| 207 | + // The held file: bytes and row kept, still `pending`, still owned. |
| 208 | + expect(await storage.exists(claimedKey), 'the held file keeps its bytes').toBe(true); |
| 209 | + expect(await fileRow(claimed)).toMatchObject({ |
| 210 | + status: 'pending', ref_object: 'media', ref_id: 'm_1', ref_field: 'poster', |
| 211 | + }); |
| 212 | + expect((await record('m_1'))?.poster).toBe(claimed); |
| 213 | + // Flagged for an operator, naming the file and its holder. |
| 214 | + const warned = logger.warn.mock.calls.map((c) => String(c[0])).filter((line) => line.includes(claimed)); |
| 215 | + expect(warned).toHaveLength(1); |
| 216 | + expect(warned[0]).toContain('media/m_1.poster'); |
| 217 | + // The unheld file of the same age: reaped, bytes and row. |
| 218 | + expect(await fileRow(abandoned), 'the abandoned upload is reaped').toBeNull(); |
| 219 | + expect(await storage.exists(abandonedKey)).toBe(false); |
| 220 | + }); |
| 221 | +}); |
0 commit comments