diff --git a/apps/roam/src/utils/__tests__/assetDegradation.test.ts b/apps/roam/src/utils/__tests__/assetDegradation.test.ts new file mode 100644 index 000000000..14ec2fca5 --- /dev/null +++ b/apps/roam/src/utils/__tests__/assetDegradation.test.ts @@ -0,0 +1,274 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { CrossAppNode } from "@repo/database/crossAppContracts"; +import type { DGSupabaseClient } from "@repo/database/lib/client"; +import { contentTypes } from "@repo/content-model"; +import type { SharedNode } from "@repo/database/lib/sharedNodes"; +import { MAX_ASSET_BYTES } from "@repo/database/lib/assetLimits"; +import { publishNodeAssets, summarizeAssetResults } from "../publishNodeAssets"; +import { importNodeAssets } from "../importNodeAssets"; +import { mirrorAssetToRoamStorage } from "../mirrorAssetToRoamStorage"; + +/** + * The degradation path, followed across both transfers rather than within one. + * + * The published markdown of the first half is the input to the second, so what a + * destination actually receives for an asset that never made it into shared storage is + * asserted rather than assumed: publication reports the failure and leaves the asset's + * link as written, and import leaves that same link alone because no row matches it. The + * two halves are covered separately in `publishNodeAssets.test.ts` and + * `importNodeAssets.test.ts`; what is only visible here is that they agree on what passes + * between them. + * + * The two halves name that string differently: publication's results carry `sourceRef`, + * the rewriter's carry `sourceLocator`. Same string, different owners, so neither name is + * wrong here and the assertions below use whichever side they are reading. + */ + +vi.mock("../mirrorAssetToRoamStorage", () => ({ + mirrorAssetToRoamStorage: vi.fn(), +})); +const mirror = vi.mocked(mirrorAssetToRoamStorage); + +/** Shared by the fixture URLs and the `graph.name` stub, which must agree: publication + * recognises an asset by the graph named in its path. */ +const GRAPH_NAME = "MAPLab"; + +const roamAsset = (name: string) => + `https://firebasestorage.googleapis.com/v0/b/firescript-577a2.appspot.com/o/imgs%2Fapp%2F${GRAPH_NAME}%2F${name}.png?alt=media&token=9f1c07a4`; + +const STORED = roamAsset("stored"); +const UNREADABLE = roamAsset("unreadable"); +const OVERSIZED = roamAsset("oversized"); +const EXTERNAL = "https://example.org/not-an-asset.png"; + +const MARKDOWN = [ + "# Sleep improves memory consolidation", + "", + `![](${STORED})`, + `![](${UNREADABLE})`, + `![](${OVERSIZED})`, + `[a paper](${EXTERNAL})`, + "", + "- Supported by [[EVD]] - Rasch & Born 2013", +].join("\n"); + +const SOURCE_LOCAL_ID = "tgWb6JozF"; + +const node: CrossAppNode = { + localId: SOURCE_LOCAL_ID, + nodeType: "rCLM0schema", + coreTitle: "Sleep improves memory consolidation", + content: { + direct: { value: "Sleep improves memory consolidation" }, + full: { contentType: contentTypes.markdown, value: MARKDOWN }, + }, + createdAt: new Date("2026-06-12T14:00:00.000Z"), + modifiedAt: new Date("2026-06-12T15:00:00.000Z"), + authorId: "maparent", +}; + +const sharedNode = { + rid: "orn:roam.node:MAPLab/tgWb6JozF", + sourceLocalId: SOURCE_LOCAL_ID, + spaceId: 20, + spaceName: "MAPLab", + spaceUri: "roam:MAPLab", + platform: "Roam", + title: "Sleep improves memory consolidation", + created: null, + lastModified: "2026-06-12T15:00:00.000Z", + directMetadata: null, +} as unknown as SharedNode; + +type Row = { + filepath: string; + filehash: string; + source_path: string | null; +}; + +/** + * One store standing in for Supabase across both halves: publication inserts into it and + * import reads back out of it, so the rows the destination sees are the rows publication + * actually wrote. + */ +const makeSharedStorage = () => { + const rows: Row[] = []; + const thenable = (result: unknown) => ({ + then: (resolve: (value: unknown) => unknown) => + Promise.resolve(result).then(resolve), + }); + const selectChain = () => { + const chain = { + eq: () => chain, + in: () => chain, + order: () => chain, + then: (resolve: (value: unknown) => unknown) => + Promise.resolve({ data: rows, error: null }).then(resolve), + }; + return chain; + }; + const client = { + rpc: vi.fn((_fn: string, { hashvalue }: { hashvalue: string }) => + Promise.resolve({ + data: rows.some((row) => row.filehash === hashvalue), + error: null, + }), + ), + storage: { + from: vi.fn(() => ({ + upload: vi.fn().mockResolvedValue({ error: null }), + })), + }, + from: vi.fn(() => ({ + select: vi.fn(() => selectChain()), + delete: vi.fn(() => { + const chain = { + eq: () => chain, + notIn: () => chain, + then: (resolve: (value: unknown) => unknown) => + Promise.resolve({ error: null }).then(resolve), + }; + return chain; + }), + insert: vi.fn((inserted: Row) => { + rows.push({ + filepath: inserted.filepath, + filehash: inserted.filehash, + source_path: inserted.source_path ?? null, + }); + return thenable({ error: null }); + }), + })), + } as unknown as DGSupabaseClient; + return { client, rows }; +}; + +/** + * Roam's storage: one asset readable, one unreadable, one past the publish cap. + * + * Both reads an asset takes are stubbed: the descriptor over `fetch`, the bytes through + * `file.get`. The unreadable one fails on either, so the test does not depend on which of + * the two happens to reach it first. + */ +const stubRoamStorage = () => { + const objectPath = (url: string) => url.split("?")[0] ?? url; + const isUnreadable = (url: string) => + objectPath(url) === objectPath(UNREADABLE); + + vi.stubGlobal( + "fetch", + vi.fn((input: string) => { + if (isUnreadable(input)) + return Promise.resolve({ + ok: false, + status: 500, + } as unknown as Response); + const size = + objectPath(input) === objectPath(OVERSIZED) ? MAX_ASSET_BYTES + 1 : 7; + return Promise.resolve({ + ok: true, + status: 200, + json: () => + Promise.resolve({ + name: "imgs/app/MAPLab/stored.png", + contentType: "image/png", + size: String(size), + metadata: { "file-name": "diagram.png" }, + }), + } as unknown as Response); + }), + ); + + const get = vi.fn(({ url }: { url: string }) => + isUnreadable(url) + ? Promise.reject(new Error("Roam could not read the file")) + : Promise.resolve( + new File(["PNGDATA"], "diagram.png", { type: "image/png" }), + ), + ); + vi.stubGlobal("window", { + roamAlphaAPI: { + file: { get }, + graph: { name: GRAPH_NAME, isEncrypted: false }, + }, + }); + return { get }; +}; + +/** What `file.upload` returns for a mirrored asset: a fresh uid in this graph's folder. */ +const THIS_GRAPHS_COPY = roamAsset("aB3dEf"); + +describe("asset degradation across both transfers", () => { + let storage: ReturnType; + + beforeEach(() => { + vi.clearAllMocks(); + storage = makeSharedStorage(); + stubRoamStorage(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + const publish = () => + publishNodeAssets({ + client: storage.client, + spaceId: 20, + nodes: [node], + }); + + it("leaves the link of every asset it could not store in the published markdown, and reports each one", async () => { + const summary = summarizeAssetResults(await publish()); + + expect(node.content.full?.value).toBe(MARKDOWN); + expect(summary.failed.map((f) => f.sourceRef)).toEqual([UNREADABLE]); + expect(summary.tooLarge.map((s) => s.sourceRef)).toEqual([OVERSIZED]); + expect(summary.copied).toBe(1); + expect(storage.rows.map((r) => r.filepath)).toEqual([STORED]); + }); + + it("imports the published markdown with its body intact, rewriting only what was stored", async () => { + await publish(); + mirror.mockResolvedValue({ + status: "mirrored", + contentHash: storage.rows[0].filehash, + url: THIS_GRAPHS_COPY, + }); + + const { markdown, report } = await importNodeAssets({ + client: storage.client, + sharedNode, + markdown: MARKDOWN, + }); + + // Only the asset that reached shared storage was mirrored, so only its locator moved. + expect(mirror).toHaveBeenCalledTimes(1); + expect(markdown).toContain(`![](${THIS_GRAPHS_COPY})`); + // The rest of the node arrives exactly as published: the two locators that never + // became rows still point at Roam's world-readable originals, which is what makes + // them render, and the external link was never ours to touch. + expect(markdown).toContain(`![](${UNREADABLE})`); + expect(markdown).toContain(`![](${OVERSIZED})`); + expect(markdown).toContain(`[a paper](${EXTERNAL})`); + expect(markdown).toContain("# Sleep improves memory consolidation"); + expect(markdown).toContain("- Supported by [[EVD]] - Rasch & Born 2013"); + expect(report).toMatchObject({ mirrored: 1, reused: 0, skipped: [] }); + }); + + it("reports an asset that fails on the way in, leaving its locator and the node body untouched", async () => { + await publish(); + mirror.mockRejectedValue(new Error("upload refused")); + + const { markdown, report } = await importNodeAssets({ + client: storage.client, + sharedNode, + markdown: MARKDOWN, + }); + + expect(markdown).toBe(MARKDOWN); + expect(report.failed).toEqual([ + { sourceLocator: STORED, message: "upload refused" }, + ]); + }); +}); diff --git a/apps/roam/src/utils/__tests__/importNodeAssets.test.ts b/apps/roam/src/utils/__tests__/importNodeAssets.test.ts new file mode 100644 index 000000000..c67d29d0a --- /dev/null +++ b/apps/roam/src/utils/__tests__/importNodeAssets.test.ts @@ -0,0 +1,402 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { DGSupabaseClient } from "@repo/database/lib/client"; +import type { SharedNode } from "@repo/database/lib/sharedNodes"; +import { importNodeAssets } from "../importNodeAssets"; +import { mirrorAssetToRoamStorage } from "../mirrorAssetToRoamStorage"; + +vi.mock("../mirrorAssetToRoamStorage", () => ({ + mirrorAssetToRoamStorage: vi.fn(), +})); + +const mirror = vi.mocked(mirrorAssetToRoamStorage); + +const IMAGE_REF = "attachments/diagram.png"; +const FILE_REF = "attachments/report.docx"; +const MIRRORED = "https://firebasestorage.googleapis.com/v0/b/f/o/x?alt=media"; + +const sharedNode = { + rid: "orn:obsidian.note:vault-a/node-1", + sourceLocalId: "node-1", + spaceId: 20, + spaceName: "Vault A", + spaceUri: "obsidian:vault-a", + platform: "Obsidian", + title: "REM sleep and recall", + created: null, + lastModified: "2026-06-14T15:00:00.000Z", + directMetadata: null, +} as unknown as SharedNode; + +type Row = { filepath: string; filehash: string; source_path: string | null }; + +const clientWithReferences = ( + rows: Row[], + error?: { message: string }, +): { client: DGSupabaseClient; from: ReturnType } => { + const result = error ? { data: null, error } : { data: rows, error: null }; + const chain = { + eq: vi.fn(() => chain), + order: vi.fn(() => chain), + then: (resolve: (value: unknown) => unknown) => + Promise.resolve(result).then(resolve), + }; + const from = vi.fn(() => ({ select: vi.fn(() => chain) })); + return { client: { from } as unknown as DGSupabaseClient, from }; +}; + +const row = ( + filepath: string, + hash: string, + sourcePath: string | null = null, +): Row => ({ + filepath, + filehash: hash, + source_path: sourcePath, +}); + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("importNodeAssets", () => { + it("points the markdown at this graph's copies and counts what it uploaded", async () => { + mirror.mockResolvedValue({ + status: "mirrored", + contentHash: "h1", + url: MIRRORED, + }); + + const result = await importNodeAssets({ + client: clientWithReferences([row(IMAGE_REF, "h1")]).client, + sharedNode, + markdown: `![](${IMAGE_REF})`, + }); + + expect(result.markdown).toBe(`![](${MIRRORED})`); + expect(result.report).toEqual({ + mirrored: 1, + reused: 0, + skipped: [], + failed: [], + }); + }); + + it("counts a copy this graph already held separately from one it uploaded", async () => { + mirror.mockResolvedValue({ + status: "reused", + contentHash: "h1", + url: MIRRORED, + }); + + const { report } = await importNodeAssets({ + client: clientWithReferences([row(IMAGE_REF, "h1")]).client, + sharedNode, + markdown: `![](${IMAGE_REF})`, + }); + + expect(report).toMatchObject({ mirrored: 0, reused: 1 }); + }); + + it("passes the recorded name through, so a non-media link can be labelled", async () => { + mirror.mockResolvedValue({ + status: "mirrored", + contentHash: "h2", + url: MIRRORED, + }); + + const result = await importNodeAssets({ + client: clientWithReferences([row(FILE_REF, "h2", "report.docx")]).client, + sharedNode, + markdown: `[](${FILE_REF})`, + }); + + expect(mirror).toHaveBeenCalledWith( + expect.objectContaining({ contentHash: "h2", sourcePath: "report.docx" }), + ); + expect(result.markdown).toBe(`[report.docx](${MIRRORED})`); + }); + + it("imports the node with its content intact when one asset fails", async () => { + mirror + .mockResolvedValueOnce({ + status: "mirrored", + contentHash: "h1", + url: MIRRORED, + }) + .mockRejectedValueOnce(new Error("upload refused")); + + const markdown = `![](${IMAGE_REF}) and [](${FILE_REF})`; + const result = await importNodeAssets({ + client: clientWithReferences([row(IMAGE_REF, "h1"), row(FILE_REF, "h2")]) + .client, + sharedNode, + markdown, + }); + + // The resolved asset is still rewritten; the failed one keeps the locator it arrived + // with, which is the degradation path rather than a broken node. + expect(result.markdown).toBe(`![](${MIRRORED}) and [](${FILE_REF})`); + expect(result.report.mirrored).toBe(1); + expect(result.report.failed).toEqual([ + { sourceLocator: FILE_REF, message: "upload refused" }, + ]); + }); + + it("reports an oversized asset and leaves its locator in place", async () => { + mirror.mockResolvedValue({ + status: "skipped", + contentHash: "h1", + reason: "too-large", + size: 9_000_000, + limit: 6_291_456, + }); + + const markdown = `![](${IMAGE_REF})`; + const result = await importNodeAssets({ + client: clientWithReferences([row(IMAGE_REF, "h1")]).client, + sharedNode, + markdown, + }); + + expect(result.markdown).toBe(markdown); + expect(result.report.skipped).toEqual([ + { + sourceLocator: IMAGE_REF, + reason: "too-large", + size: 9_000_000, + limit: 6_291_456, + }, + ]); + }); + + it("imports a node whose every asset fails, reporting each one", async () => { + mirror.mockRejectedValue(new Error("storage unreachable")); + + const markdown = `![](${IMAGE_REF}) and [](${FILE_REF})`; + const result = await importNodeAssets({ + client: clientWithReferences([row(IMAGE_REF, "h1"), row(FILE_REF, "h2")]) + .client, + sharedNode, + markdown, + }); + + expect(result.markdown).toBe(markdown); + expect(result.report.failed).toHaveLength(2); + }); + + it("does not fail the node when the references cannot be read", async () => { + const markdown = `![](${IMAGE_REF})`; + const result = await importNodeAssets({ + client: clientWithReferences([], { message: "permission denied" }).client, + sharedNode, + markdown, + }); + + expect(result.markdown).toBe(markdown); + expect(result.report.failed).toEqual([ + { + message: expect.stringContaining("permission denied") as string, + }, + ]); + // `toEqual` cannot tell an absent optional property from an explicit undefined, so + // the absence is asserted directly: this failure is about the node, not one asset. + expect(result.report.failed[0]).not.toHaveProperty("sourceLocator"); + expect(mirror).not.toHaveBeenCalled(); + }); + + // A node published from Roam and imported into a second Roam graph. The locator is a URL + // the importing graph could render directly, and it is still resolved through its row + // and copied: recognising a storage URL in order to skip the copy would put origin + // detection back into the destination, and it would leave this graph's page depending + // on a blob the origin graph's owner can delete. + it("stores its own copy of a Roam-origin asset rather than passing the origin URL through", async () => { + const originUrl = + "https://firebasestorage.googleapis.com/v0/b/firescript-577a2.appspot.com/o/imgs%2Fapp%2FOriginGraph%2FlqP2ioVNC3.png?alt=media&token=9f1c07a4"; + mirror.mockResolvedValue({ + status: "mirrored", + contentHash: "h1", + url: MIRRORED, + }); + + const result = await importNodeAssets({ + client: clientWithReferences([ + row(originUrl, "h1", "CleanShot 2025-11-16 at 17.14.44@2x.png"), + ]).client, + sharedNode, + markdown: `![](${originUrl})`, + }); + + expect(mirror).toHaveBeenCalledTimes(1); + expect(result.markdown).toBe(`![](${MIRRORED})`); + expect(result.markdown).not.toContain("OriginGraph"); + expect(result.report.mirrored).toBe(1); + }); + + it("copies nothing for a node with no recorded references", async () => { + const markdown = `![](${IMAGE_REF})`; + const result = await importNodeAssets({ + client: clientWithReferences([]).client, + sharedNode, + markdown, + }); + + expect(result.markdown).toBe(markdown); + expect(result.report).toEqual({ + mirrored: 0, + reused: 0, + skipped: [], + failed: [], + }); + expect(mirror).not.toHaveBeenCalled(); + }); + + it("hands each caller its own report, so one node's failure is not another's", async () => { + const first = await importNodeAssets({ + client: clientWithReferences([]).client, + sharedNode, + markdown: "no assets here", + }); + const second = await importNodeAssets({ + client: clientWithReferences([]).client, + sharedNode, + markdown: "none here either", + }); + + first.report.failed.push({ sourceLocator: "x", message: "mine alone" }); + + expect(second.report.failed).toEqual([]); + }); + + it("counts one upload when two locators name identical bytes", async () => { + mirror + .mockResolvedValueOnce({ + status: "mirrored", + contentHash: "h1", + url: MIRRORED, + }) + .mockResolvedValueOnce({ + status: "reused", + contentHash: "h1", + url: MIRRORED, + }); + + const { report } = await importNodeAssets({ + client: clientWithReferences([ + row(IMAGE_REF, "h1"), + row("attachments/copy.png", "h1"), + ]).client, + sharedNode, + markdown: `![](${IMAGE_REF}) ![](attachments/copy.png)`, + }); + + // Not `reused: 1`: this run uploaded those bytes itself a moment earlier, and a first + // import reporting a cache hit would be a lie about where the copy came from. + expect(report).toMatchObject({ mirrored: 1, reused: 0 }); + // Both references are still mirrored. Deduplication belongs to the registry inside + // `mirrorAssetToRoamStorage`, which is what turns the second call into a reuse; this + // module's job is only to count blobs rather than locators. + expect(mirror).toHaveBeenCalledTimes(2); + }); + + it("asks once whether a blob is oversized, and reports every locator naming it", async () => { + mirror.mockResolvedValue({ + status: "skipped", + contentHash: "h1", + reason: "too-large", + size: 9_000_000, + limit: 6_291_456, + }); + + const markdown = `![](${IMAGE_REF}) ![](attachments/copy.png)`; + const { report } = await importNodeAssets({ + client: clientWithReferences([ + row(IMAGE_REF, "h1"), + row("attachments/copy.png", "h1"), + ]).client, + sharedNode, + markdown, + }); + + // Decided once per hash, so the second locator adds no round trip. + expect(mirror).toHaveBeenCalledTimes(1); + // Still one entry per locator. + expect(report.skipped).toEqual([ + { + sourceLocator: IMAGE_REF, + reason: "too-large", + size: 9_000_000, + limit: 6_291_456, + }, + { + sourceLocator: "attachments/copy.png", + reason: "too-large", + size: 9_000_000, + limit: 6_291_456, + }, + ]); + }); + + it("copies nothing for a reference the fetched markdown never makes", async () => { + const { client } = clientWithReferences([ + row("attachments/only-in-frontmatter.png", "h1"), + ]); + + const { report } = await importNodeAssets({ + client, + sharedNode, + markdown: "A body that mentions no assets at all.", + }); + + // The row outlived its locator: stripped frontmatter, or a publish whose best-effort + // cleanup failed. Uploading it would spend the user's storage permanently on bytes + // no block can reference. + expect(mirror).not.toHaveBeenCalled(); + expect(report).toMatchObject({ mirrored: 0, reused: 0 }); + }); + + it("copies a reference the markdown percent-encodes", async () => { + mirror.mockResolvedValue({ + status: "mirrored", + contentHash: "h1", + url: MIRRORED, + }); + + const result = await importNodeAssets({ + client: clientWithReferences([row("my folder/d.png", "h1")]).client, + sharedNode, + markdown: `![](my%20folder/d.png)`, + }); + + expect(mirror).toHaveBeenCalledTimes(1); + expect(result.markdown).toBe(`![](${MIRRORED})`); + }); + + it("copies a reference whose name forces an encoding encodeURI would not apply", async () => { + // `fig#1.png` is written `fig%231.png`, because `#` starts a fragment. Deriving the + // spellings forward would miss it and drop the asset; reading the locators the rewriter + // will act on cannot, because it is the same set. + mirror.mockResolvedValue({ + status: "mirrored", + contentHash: "h1", + url: MIRRORED, + }); + + const result = await importNodeAssets({ + client: clientWithReferences([row("fig#1.png", "h1")]).client, + sharedNode, + markdown: `![](fig%231.png)`, + }); + + expect(mirror).toHaveBeenCalledTimes(1); + expect(result.markdown).toBe(`![](${MIRRORED})`); + }); + + it("copies nothing for a node with no content", async () => { + const { client, from } = clientWithReferences([row(IMAGE_REF, "h1")]); + const result = await importNodeAssets({ client, sharedNode, markdown: "" }); + + expect(result.markdown).toBe(""); + // Not even the reference query runs: there is nothing a rewrite could apply to. + expect(from).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/roam/src/utils/__tests__/importSharedNodes.test.ts b/apps/roam/src/utils/__tests__/importSharedNodes.test.ts index 0712131d8..c2247d45c 100644 --- a/apps/roam/src/utils/__tests__/importSharedNodes.test.ts +++ b/apps/roam/src/utils/__tests__/importSharedNodes.test.ts @@ -9,12 +9,12 @@ import { import { materializeSharedNode } from "~/utils/materializeSharedNode"; import { resolveSharedNodeTypes } from "~/utils/resolveSharedNodeTypes"; -vi.mock("~/utils/materializeSharedNode", async () => { - const actual = await vi.importActual< - typeof import("~/utils/materializeSharedNode") - >("~/utils/materializeSharedNode"); - return { ...actual, materializeSharedNode: vi.fn() }; -}); +// A plain factory rather than `importActual`: loading the real module pulls in +// `internalError`, whose settings-accessor chain reads `window` at module scope. Nothing +// reachable from this test needs the module's other exports. +vi.mock("~/utils/materializeSharedNode", () => ({ + materializeSharedNode: vi.fn(), +})); vi.mock("~/utils/resolveSharedNodeTypes", () => ({ resolveSharedNodeTypes: vi.fn(), diff --git a/apps/roam/src/utils/__tests__/rewriteAssetLinks.test.ts b/apps/roam/src/utils/__tests__/rewriteAssetLinks.test.ts index 3456226fa..9dbd20f50 100644 --- a/apps/roam/src/utils/__tests__/rewriteAssetLinks.test.ts +++ b/apps/roam/src/utils/__tests__/rewriteAssetLinks.test.ts @@ -3,7 +3,11 @@ import { obsidianOriginNodeExample, roamOriginNodeExample, } from "@repo/database/crossAppNodeContract.example"; -import { rewriteAssetLinks } from "../rewriteAssetLinks"; +import { + collectAssetLocators, + lookupCandidates, + rewriteAssetLinks, +} from "../rewriteAssetLinks"; const MIRRORED = "https://firebasestorage.googleapis.com/v0/b/f.appspot.com/o/x?alt=media&token=abc"; @@ -562,3 +566,100 @@ describe("the cross-app contract fixtures round-trip", () => { ); }); }); + +/** + * Pins what `collectAssetLocators` promises: the locators a caller can see are the ones + * the rewrite will act on. + * + * The corpus carries one of every branch `LINK_PATTERN` lists, so a capture-group change + * that leaves the rewriter working still fails here. Nothing else would catch it, because + * a divergence imports the node successfully, just wrong. + */ +describe("collectAssetLocators reads what rewriteAssetLinks acts on", () => { + const CORPUS = [ + `![a diagram](vault/a.png)`, + `[the report](vault/b.docx)`, + `{{[[pdf]]: https://storage.test/c.pdf}}`, + `{{audio: https://storage.test/d.mp3}}`, + `![[vault/e.png]]`, + `[[vault/f.png|Figure 6]]`, + `See https://storage.test/g.png, then stop.`, + `![](my%20folder/h.png)`, + `![[vault/i.docx|a label]]`, + `![]()`, + ``, + `A page reference, [[EVD]], which no row matches.`, + ].join("\n\n"); + + // As a `FileReference` records them, which is not always as the markdown spells them: + // the bare URL arrives with the sentence's comma attached, and Obsidian records a + // vault path decoded. + const RECORDED = [ + "vault/a.png", + "vault/b.docx", + "https://storage.test/c.pdf", + "https://storage.test/d.mp3", + "vault/e.png", + "vault/f.png", + "https://storage.test/g.png", + "my folder/h.png", + "vault/i.docx", + "my folder/j.png", + "https://storage.test/k.png", + ]; + + it("collects one locator per reference, in the order the markdown makes them", () => { + // Written out rather than derived, so a drift in the capture groups fails here + // instead of being absorbed by whatever derived it. + expect(collectAssetLocators(CORPUS)).toEqual([ + "vault/a.png", + "vault/b.docx", + "https://storage.test/c.pdf", + "https://storage.test/d.mp3", + "vault/e.png", + "vault/f.png", + "https://storage.test/g.png,", + "my%20folder/h.png", + // The embed's alias is a group of its own, so it is never a locator. + "vault/i.docx", + // Angle brackets delimit the locator, so they are gone by the time it is collected. + "my folder/j.png", + "https://storage.test/k.png", + "EVD", + ]); + }); + + it("reaches every recorded locator once widened, which is what the caller filters on", () => { + const resolvable = new Set( + collectAssetLocators(CORPUS).flatMap(lookupCandidates), + ); + for (const recorded of RECORDED) expect(resolvable).toContain(recorded); + }); + + it("rewrites every locator it collected, and nothing it did not", () => { + const assets = RECORDED.map((sourceLocator, index) => ({ + sourceLocator, + url: `https://mirror.test/${index}`, + })); + const result = rewriteAssetLinks({ markdown: CORPUS, assets }); + + // Each asset reached its own copy, so no branch was collected but left unrewritten. + for (const asset of assets) expect(result).toContain(asset.url); + // And no original spelling survived, so none was rewritten only in part. + for (const spelling of [ + "vault/a.png", + "vault/b.docx", + "vault/e.png", + "vault/f.png", + "my%20folder/h.png", + "vault/i.docx", + "my folder/j.png", + "storage.test", + ]) + expect(result).not.toContain(spelling); + + // `EVD` is collected like any other match, because this file cannot know which + // locators have rows. Having no row is what leaves it alone. + expect(result).toContain(`[[EVD]]`); + }); +}); diff --git a/apps/roam/src/utils/getErrorMessage.ts b/apps/roam/src/utils/getErrorMessage.ts new file mode 100644 index 000000000..6fb32814f --- /dev/null +++ b/apps/roam/src/utils/getErrorMessage.ts @@ -0,0 +1,17 @@ +/** + * The message to show for something that was thrown, wherever it came from. + * + * Shared by `materializeSharedNode` and the asset stage it calls. In either file, the + * other would have to import its caller. + * + * Supabase reports a failed query as a plain object carrying `message`, not an `Error`, + * so read both shapes. Stringifying the object yields `[object Object]`. + */ +export const getErrorMessage = (error: unknown): string => { + if (error instanceof Error) return error.message; + if (typeof error === "object" && error !== null && "message" in error) { + const { message } = error; + if (typeof message === "string") return message; + } + return String(error); +}; diff --git a/apps/roam/src/utils/importNodeAssets.ts b/apps/roam/src/utils/importNodeAssets.ts new file mode 100644 index 000000000..d9871819d --- /dev/null +++ b/apps/roam/src/utils/importNodeAssets.ts @@ -0,0 +1,231 @@ +import type { DGSupabaseClient } from "@repo/database/lib/client"; +import type { SharedNode } from "@repo/database/lib/sharedNodes"; +import { getErrorMessage } from "./getErrorMessage"; +import { mirrorAssetToRoamStorage } from "./mirrorAssetToRoamStorage"; +import { + collectAssetLocators, + lookupCandidates, + rewriteAssetLinks, + type ResolvedAsset, +} from "./rewriteAssetLinks"; + +export type SkippedImport = { + sourceLocator: string; + reason: "too-large"; + size: number; + limit: number; +}; + +export type FailedImport = { + /** Absent when the failure was not about one asset. */ + sourceLocator?: string; + message: string; +}; + +export type AssetImportReport = { + /** Uploaded into this graph's storage by this run. */ + mirrored: number; + /** Already held by this graph, so no upload was needed. */ + reused: number; + skipped: SkippedImport[]; + failed: FailedImport[]; +}; + +/** + * A fresh report per call, never a shared constant. Callers own what they are handed and + * the arrays are mutable, so one `report.failed.push(...)` on a returned object would + * otherwise attribute one node's failure to every asset-free node in the session. + */ +const emptyReport = (): AssetImportReport => ({ + mirrored: 0, + reused: 0, + skipped: [], + failed: [], +}); + +type ReferenceRow = { + filepath: string; + filehash: string; + source_path: string | null; +}; + +/** + * The references recorded against the published node, which are the only things this + * stage resolves. A node with no rows has no assets to copy, whether because it + * references none or because none could be stored when it was published. + */ +const fetchNodeReferences = async ({ + client, + sharedNode, +}: { + client: DGSupabaseClient; + sharedNode: SharedNode; +}): Promise => { + const { data, error } = await client + .from("my_file_references") + .select("filepath, filehash, source_path") + .eq("space_id", sharedNode.spaceId) + .eq("source_local_id", sharedNode.sourceLocalId) + // Ordered so a repeated import does the same thing twice. Where two references share + // a hash, the first one mirrored decides the uploaded file's extension, because the + // second reuses its URL; without an order, which name that is comes down to whatever + // Postgres returned first. + .order("filepath"); + if (error) throw error; + return (data ?? []).flatMap((row): ReferenceRow[] => + typeof row.filepath === "string" && typeof row.filehash === "string" + ? [ + { + filepath: row.filepath, + filehash: row.filehash, + source_path: + typeof row.source_path === "string" ? row.source_path : null, + }, + ] + : [], + ); +}; + +/** + * The asset stage of materialization: copy the bytes an imported node references into + * this graph's storage, and point the node's markdown at those copies. + * + * It runs between fetching the content and replacing the page's blocks, because the + * markdown it returns is what gets written. Nothing here can fail the node: an asset that + * cannot be copied leaves its locator exactly as published, and is reported instead of + * thrown. + * + * What a surviving locator does depends on its origin. A Roam-origin locator is a public + * Firebase URL, so the block still renders from the origin graph. An Obsidian-origin + * locator is a vault path, and Roam reads `![[attachments/diagram.png]]` as a page + * reference, so a failed Obsidian asset leaves a link to an empty page. + * + * That is deliberate. An unresolved locator must survive unchanged so a later re-import + * can resolve it. + */ +export const importNodeAssets = async ({ + client, + sharedNode, + markdown, +}: { + client: DGSupabaseClient; + sharedNode: SharedNode; + markdown: string; +}): Promise<{ markdown: string; report: AssetImportReport }> => { + if (!markdown) return { markdown, report: emptyReport() }; + + let references: ReferenceRow[]; + try { + references = await fetchNodeReferences({ client, sharedNode }); + } catch (error) { + // The node still imports, with every asset locator left as published. Reported as one + // failure rather than none, because "no rows" and "could not read the rows" produce + // the same content and must not look the same to a reader. + return { + markdown, + report: { + ...emptyReport(), + failed: [ + { + message: `Could not read the asset references of "${sharedNode.title}": ${getErrorMessage(error)}`, + }, + ], + }, + }; + } + if (!references.length) return { markdown, report: emptyReport() }; + + // The locators come from the rewriter's own reading of the text, not from re-deriving + // the spellings a path might take. Generating them forward cannot work: a note writes + // `fig#1.png` as `fig%231.png`, and `encodeURI` leaves `#` and `?` alone, so a filter + // built that way drops an asset the rewrite would have resolved. + const resolvable = new Set( + collectAssetLocators(markdown).flatMap(lookupCandidates), + ); + + // Only the references this content actually makes. A row can outlive its locator two + // ways: `publishNodeAssets` cleans stale rows best-effort and logs rather than fails, + // and the markdown fetched here has had its frontmatter or title heading stripped, so + // an asset referenced only there has a row and no locator. Copying one would spend the + // user's storage, permanently, on bytes no block will ever point at. + const referenced = references.filter(({ filepath }) => + resolvable.has(filepath), + ); + if (!referenced.length) return { markdown, report: emptyReport() }; + + const resolved: ResolvedAsset[] = []; + const report = emptyReport(); + /** + * Counts are per distinct blob, not per reference. Two locators for identical bytes are + * one upload, and reporting the second as `reused` would tell a user on a first-ever + * import that this graph already held something it had just fetched. + * + * `skipped` and `failed` stay per locator: two references to one oversized blob are + * two places the page degraded. + */ + const handledHashes = new Set(); + /** + * Oversize is a property of the bytes, so it is decided once per hash. Asking again + * costs a `storage.info` round trip, and where the object carries no size, a second + * download of a blob already known to be over the cap. + * + * A throw is not cached: it can be a transient read failure rather than a fact about + * the asset, so a second locator may try again. + */ + const skippedByHash = new Map(); + + // Sequential on purpose. Two references to identical content share a hash, and the + // registry is what stops the second one uploading again; running them together would + // race that check and mirror the same bytes twice. + for (const reference of referenced) { + const alreadySkipped = skippedByHash.get(reference.filehash); + if (alreadySkipped) { + report.skipped.push({ + sourceLocator: reference.filepath, + reason: "too-large", + ...alreadySkipped, + }); + continue; + } + try { + const result = await mirrorAssetToRoamStorage({ + client, + contentHash: reference.filehash, + sourcePath: reference.source_path, + }); + if (result.status === "skipped") { + skippedByHash.set(reference.filehash, { + size: result.size, + limit: result.limit, + }); + report.skipped.push({ + sourceLocator: reference.filepath, + reason: result.reason, + size: result.size, + limit: result.limit, + }); + continue; + } + if (!handledHashes.has(reference.filehash)) { + handledHashes.add(reference.filehash); + if (result.status === "mirrored") report.mirrored += 1; + else report.reused += 1; + } + resolved.push({ + sourceLocator: reference.filepath, + url: result.url, + sourcePath: reference.source_path, + }); + } catch (error) { + report.failed.push({ + sourceLocator: reference.filepath, + message: getErrorMessage(error), + }); + } + } + + return { + markdown: rewriteAssetLinks({ markdown, assets: resolved }), + report, + }; +}; diff --git a/apps/roam/src/utils/importSharedNodes.ts b/apps/roam/src/utils/importSharedNodes.ts index 6a4f23abc..0d9c3931b 100644 --- a/apps/roam/src/utils/importSharedNodes.ts +++ b/apps/roam/src/utils/importSharedNodes.ts @@ -1,9 +1,7 @@ import type { DGSupabaseClient } from "@repo/database/lib/client"; import type { SharedNode } from "@repo/database/lib/sharedNodes"; -import { - getErrorMessage, - materializeSharedNode, -} from "./materializeSharedNode"; +import { getErrorMessage } from "./getErrorMessage"; +import { materializeSharedNode } from "./materializeSharedNode"; import { resolveSharedNodeTypes } from "./resolveSharedNodeTypes"; export type FailedSharedNodeImport = { diff --git a/apps/roam/src/utils/materializeSharedNode.ts b/apps/roam/src/utils/materializeSharedNode.ts index 6fbd7cac0..3740a72bc 100644 --- a/apps/roam/src/utils/materializeSharedNode.ts +++ b/apps/roam/src/utils/materializeSharedNode.ts @@ -19,6 +19,7 @@ import { writeImportedSourceIdentity, type ImportedSourceIdentity, } from "./importedSourceIdentity"; +import { getErrorMessage } from "./getErrorMessage"; type MaterializationStage = | "validate-input" @@ -72,9 +73,6 @@ type RoamMarkdownApi = { export const getRoamMarkdownApi = (): RoamMarkdownApi => window.roamAlphaAPI.data as unknown as RoamMarkdownApi; -export const getErrorMessage = (error: unknown): string => - error instanceof Error ? error.message : String(error); - const isImportUpToDate = ({ sourceModifiedAt, storedModifiedAt, diff --git a/apps/roam/src/utils/mirrorAssetToRoamStorage.ts b/apps/roam/src/utils/mirrorAssetToRoamStorage.ts index df2e94f3c..49adca592 100644 --- a/apps/roam/src/utils/mirrorAssetToRoamStorage.ts +++ b/apps/roam/src/utils/mirrorAssetToRoamStorage.ts @@ -4,6 +4,7 @@ import { } from "@repo/database/lib/assetLimits"; import type { DGSupabaseClient } from "@repo/database/lib/client"; import { readMirroredAssetUrl, recordMirroredAsset } from "./assetRegistry"; +import { getErrorMessage } from "./getErrorMessage"; /** * Copies one asset out of shared storage and into this graph's own Roam storage. @@ -18,6 +19,14 @@ import { readMirroredAssetUrl, recordMirroredAsset } from "./assetRegistry"; * detection back into the destination, and would leave this graph's page depending on a * blob the origin graph's owner can delete. The copy is irrevocable, deliberately. * + * **Nothing rolls an upload back.** `file.delete` exists and takes a URL, so a caller + * holding one could undo its own upload, but it must not: a graph's users share one + * registry keyed by content hash (see `assetRegistry`), so a blob this call uploaded may + * already have been resolved by another user's import. A failed import leaves its copies + * in place instead, and callers order their work so a rejected import never uploads at + * all. Roam exposes no way to list a graph's files, so an orphan cannot be swept up + * afterwards either. + * * **Call this one asset at a time.** The registry read and the matching write are * separated by a download and an upload, so callers running it under `Promise.all` all * see an empty registry for the same hash: the bytes upload once per call, the registry @@ -26,9 +35,6 @@ import { readMirroredAssetUrl, recordMirroredAsset } from "./assetRegistry"; * in-flight map of hash to promise here first. */ -const getErrorMessage = (error: unknown): string => - error instanceof Error ? error.message : String(error); - /** The bucket `addFile` writes to, keyed by content hash. */ const SHARED_ASSET_BUCKET = "assets"; diff --git a/apps/roam/src/utils/refreshImportedNode.ts b/apps/roam/src/utils/refreshImportedNode.ts index a3ae9208a..1a4b668bd 100644 --- a/apps/roam/src/utils/refreshImportedNode.ts +++ b/apps/roam/src/utils/refreshImportedNode.ts @@ -2,10 +2,8 @@ import { getSharedNodeByRid } from "@repo/database/lib/sharedNodes"; import getPageTitleByPageUid from "roamjs-components/queries/getPageTitleByPageUid"; import { readImportedSourceIdentity } from "./importedSourceIdentity"; import internalError from "./internalError"; -import { - getErrorMessage, - materializeSharedNode, -} from "./materializeSharedNode"; +import { getErrorMessage } from "./getErrorMessage"; +import { materializeSharedNode } from "./materializeSharedNode"; import { resolveSharedNodeTypes } from "./resolveSharedNodeTypes"; import { getLoggedInClient } from "./supabaseContext"; diff --git a/apps/roam/src/utils/rewriteAssetLinks.ts b/apps/roam/src/utils/rewriteAssetLinks.ts index 374651304..5f179e88b 100644 --- a/apps/roam/src/utils/rewriteAssetLinks.ts +++ b/apps/roam/src/utils/rewriteAssetLinks.ts @@ -274,7 +274,7 @@ const LINK_PATTERN = new RegExp( * `metadataCache`, while the note itself holds `my%20folder/d.png`, so any vault path * with a space in it needs the decoded form to match. */ -const lookupCandidates = (locator: string): string[] => { +export const lookupCandidates = (locator: string): string[] => { const candidates = [locator]; const withoutPunctuation = locator.replace(TRAILING_PUNCTUATION, ""); if (withoutPunctuation !== locator) candidates.push(withoutPunctuation); @@ -289,7 +289,14 @@ const lookupCandidates = (locator: string): string[] => { return candidates; }; -// One capture group per branch, in the order the pattern lists them. +/** + * What one match of `LINK_PATTERN` refers to, read from the capture groups in the order + * the pattern lists its branches. + * + * Shared with `collectAssetLocators` so that the locators a caller can see are exactly the + * locators this file will rewrite. Anything deriving that set independently drifts from it, + * and a locator missing from the caller's set is an asset silently dropped. + */ const parseMatch = ( groups: (string | undefined)[], ): @@ -350,6 +357,24 @@ const parseMatch = ( }; }; +/** + * Every locator this markdown refers an asset by, as `rewriteAssetLinks` will read them. + * + * A caller deciding which recorded references are worth acting on has to ask the text the + * same question the rewrite will ask it. Widening each of these through + * `lookupCandidates` yields exactly the set of `locator` values that would resolve, so + * a caller's set and the rewriter's are equal by construction rather than by agreement. + */ +export const collectAssetLocators = (markdown: string): string[] => { + const locators: string[] = []; + for (const match of markdown.matchAll(LINK_PATTERN)) { + const [, ...groups] = match; + const parsed = parseMatch(groups); + if (parsed) locators.push(parsed.locator); + } + return locators; +}; + export const rewriteAssetLinks = ({ markdown, assets,