diff --git a/apps/obsidian/src/utils/__tests__/importedNodeContent.test.ts b/apps/obsidian/src/utils/__tests__/importedNodeContent.test.ts new file mode 100644 index 0000000000..5513bc3d60 --- /dev/null +++ b/apps/obsidian/src/utils/__tests__/importedNodeContent.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "vitest"; +import { + type ImportedNodeContentRow, + resolveImportedNodeContent, +} from "~/utils/importedNodeContent"; + +const directRow = ( + overrides: Partial = {}, +): ImportedNodeContentRow => ({ + variant: "direct", + text: "Note title", + created: "2026-01-01T00:00:00", + last_modified: "2026-01-02T00:00:00", + author_id: 7, + metadata: {}, + ...overrides, +}); + +const fullRow = ( + overrides: Partial = {}, +): ImportedNodeContentRow => ({ + variant: "full", + text: "body", + created: "2026-01-01T00:00:00", + last_modified: "2026-01-02T00:00:00", + author_id: 7, + metadata: {}, + ...overrides, +}); + +describe("resolveImportedNodeContent", () => { + it("pairs the direct title with the full body", () => { + expect(resolveImportedNodeContent([directRow(), fullRow()])).toMatchObject({ + fileName: "Note title", + content: "body", + authorId: 7, + createdAt: new Date("2026-01-01T00:00:00Z").valueOf(), + modifiedAt: new Date("2026-01-02T00:00:00Z").valueOf(), + }); + }); + + it("accepts an empty body, which a titled node with no content publishes", () => { + expect( + resolveImportedNodeContent([directRow(), fullRow({ text: "" })]), + ).toMatchObject({ fileName: "Note title", content: "" }); + }); + + it("rejects a full row whose text is absent", () => { + expect( + resolveImportedNodeContent([directRow(), fullRow({ text: null })]), + ).toBeNull(); + }); + + it("rejects a node with no full row", () => { + expect(resolveImportedNodeContent([directRow()])).toBeNull(); + }); + + it("rejects a node with no title", () => { + expect( + resolveImportedNodeContent([directRow({ text: "" }), fullRow()]), + ).toBeNull(); + }); + + it("rejects a full row with no timestamps", () => { + expect( + resolveImportedNodeContent([ + directRow(), + fullRow({ created: null, last_modified: null }), + ]), + ).toBeNull(); + }); + + it("falls back to the direct row's author when the full row has none", () => { + expect( + resolveImportedNodeContent([directRow(), fullRow({ author_id: null })]), + ).toMatchObject({ authorId: 7 }); + }); + + it("rejects a node with no author on either row", () => { + expect( + resolveImportedNodeContent([ + directRow({ author_id: null }), + fullRow({ author_id: null }), + ]), + ).toBeNull(); + }); + + it("reads the source vault path from the direct row's metadata", () => { + expect( + resolveImportedNodeContent([ + directRow({ metadata: { filePath: "Notes/note.md" } }), + fullRow(), + ]), + ).toMatchObject({ filePath: "Notes/note.md" }); + }); + + it("leaves the path undefined when the metadata is an array", () => { + expect( + resolveImportedNodeContent([ + directRow({ metadata: ["Notes/note.md"] }), + fullRow(), + ]), + ).toMatchObject({ filePath: undefined }); + }); + + it("leaves the path undefined when the metadata does not carry one", () => { + expect( + resolveImportedNodeContent([ + directRow({ metadata: { filePath: 3 } }), + fullRow(), + ]), + ).toMatchObject({ filePath: undefined }); + }); +}); diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 41fdab28d8..db113f88f8 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -1,4 +1,3 @@ -import type { Json } from "@repo/database/dbTypes"; import matter from "gray-matter"; import { App, Notice, TFile } from "obsidian"; import type { DGSupabaseClient } from "@repo/database/lib/client"; @@ -32,6 +31,10 @@ import { import { createTemplateFile } from "./templates"; import { resolveFolderForSpaceUri } from "./importFolderMetadata"; import { getNodeTypeById, isAcceptedSchema } from "./typeUtils"; +import { + type ImportedNodeContent, + resolveImportedNodeContent, +} from "./importedNodeContent"; import { decorateTitle } from "@repo/database/lib/decorateTitle"; type PublishedNode = { @@ -261,14 +264,7 @@ const fetchNodeContentForImport = async ({ client: DGSupabaseClient; spaceId: number; nodeInstanceId: string; -}): Promise<{ - fileName: string; - content: string; - createdAt: number; - modifiedAt: number; - authorId: number; - filePath?: string; -} | null> => { +}): Promise => { const { data, error } = await client .from("my_contents") .select("text, created, last_modified, variant, metadata, author_id") @@ -281,41 +277,7 @@ const fetchNodeContentForImport = async ({ return null; } - const rows = (data ?? []) as Array<{ - text: string | null; - created: string | null; - last_modified: string | null; - author_id: number | null; - variant: string | null; - metadata: Json; - }>; - const direct = rows.find((r) => r.variant === "direct"); - const full = rows.find((r) => r.variant === "full"); - const authorId = full?.author_id ?? direct?.author_id ?? null; - - if ( - !direct?.text || - !full?.text || - full.created === null || - full.last_modified === null || - authorId === null - ) { - return null; - } - - const filePath: string | undefined = - typeof direct.metadata === "object" && - typeof (direct.metadata as Record).filePath === "string" - ? (direct.metadata as Record).filePath - : undefined; - return { - fileName: direct.text, - content: full.text, - createdAt: new Date(full.created + "Z").valueOf(), - modifiedAt: new Date(full.last_modified + "Z").valueOf(), - filePath, - authorId, - }; + return resolveImportedNodeContent(data ?? []); }; type NodeTypeSchemaForInstance = { diff --git a/apps/obsidian/src/utils/importedNodeContent.ts b/apps/obsidian/src/utils/importedNodeContent.ts new file mode 100644 index 0000000000..23ea4d8713 --- /dev/null +++ b/apps/obsidian/src/utils/importedNodeContent.ts @@ -0,0 +1,54 @@ +import type { Json } from "@repo/database/dbTypes"; + +export type ImportedNodeContentRow = { + text: string | null; + created: string | null; + last_modified: string | null; + author_id: number | null; + variant: string | null; + metadata: Json; +}; + +export type ImportedNodeContent = { + fileName: string; + content: string; + createdAt: number; + modifiedAt: number; + authorId: number; + filePath?: string; +}; + +const readFilePath = (metadata: Json): string | undefined => { + if (typeof metadata !== "object" || metadata === null) return undefined; + const filePath = (metadata as Record).filePath; + return typeof filePath === "string" ? filePath : undefined; +}; + +// An empty `full` body is valid: a titled node with no content publishes one, +// so only an absent row or a null text rejects the node. +export const resolveImportedNodeContent = ( + rows: ImportedNodeContentRow[], +): ImportedNodeContent | null => { + const direct = rows.find((row) => row.variant === "direct"); + const full = rows.find((row) => row.variant === "full"); + const authorId = full?.author_id ?? direct?.author_id ?? null; + + if ( + !direct?.text || + full?.text == null || + full.created === null || + full.last_modified === null || + authorId === null + ) { + return null; + } + + return { + fileName: direct.text, + content: full.text, + createdAt: new Date(full.created + "Z").valueOf(), + modifiedAt: new Date(full.last_modified + "Z").valueOf(), + filePath: readFilePath(direct.metadata), + authorId, + }; +}; diff --git a/apps/roam/src/utils/__tests__/buildFullMarkdown.test.ts b/apps/roam/src/utils/__tests__/buildFullMarkdown.test.ts new file mode 100644 index 0000000000..cb9877eb9e --- /dev/null +++ b/apps/roam/src/utils/__tests__/buildFullMarkdown.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it, vi } from "vitest"; +import type { TreeNode } from "roamjs-components/types"; + +vi.mock("roamjs-components/queries/getFullTreeByParentUid", () => ({ + default: () => ({ children: [] }), +})); +vi.mock("roamjs-components/queries/getPageViewType", () => ({ + default: () => "bullet", +})); +vi.mock("~/utils/pageToMarkdown", () => ({ + toMarkdown: ({ c }: { c: TreeNode }) => `- ${c.text}`, +})); + +// Runs before the imports below: getDiscourseNodes calls generateUID at module load. +vi.hoisted(() => { + (globalThis as { window?: unknown }).window = { + roamAlphaAPI: { util: { generateUID: () => "someUid" } }, + }; +}); + +import { buildFullMarkdown } from "~/utils/roamToCrossAppConverters"; + +const block = (text: string): TreeNode => + ({ text, children: [], order: 0, uid: "" }) as unknown as TreeNode; + +describe("buildFullMarkdown", () => { + it("emits the body without the page title", () => { + expect( + buildFullMarkdown({ blocks: [block("first"), block("second")] }), + ).toBe("- first\n- second\n"); + }); + + it("emits an empty string for a page with no blocks", () => { + expect(buildFullMarkdown({ blocks: [] })).toBe(""); + }); + + it("ignores blocks that are empty and childless", () => { + expect(buildFullMarkdown({ blocks: [block(""), block("kept")] })).toBe( + "- kept\n", + ); + }); +}); diff --git a/apps/roam/src/utils/__tests__/materializeSharedNode.test.ts b/apps/roam/src/utils/__tests__/materializeSharedNode.test.ts index b0bfb5d3b5..c8b8c85252 100644 --- a/apps/roam/src/utils/__tests__/materializeSharedNode.test.ts +++ b/apps/roam/src/utils/__tests__/materializeSharedNode.test.ts @@ -833,9 +833,9 @@ describe("materializeSharedNode", () => { }); }); - it("strips the Roam heading by the source title while decorating the page title", async () => { + it("keeps the Roam body intact while decorating the page title", async () => { const { client } = clientWithFullContent({ - text: `# ${roamSharedNode.title}\n\n- REM sleep improves recall`, + text: "- REM sleep improves recall", contentType: "text/roam+markdown", }); @@ -989,9 +989,9 @@ describe("materializeSharedNode", () => { expect(mockedImportNodeAssets).toHaveBeenCalled(); }); - it("imports a Roam-origin node and strips the duplicated title heading", async () => { + it("imports a Roam-origin node body verbatim", async () => { const { client } = clientWithFullContent({ - text: `# ${roamSharedNode.title}\n\n- REM sleep improves recall\n`, + text: "- REM sleep improves recall\n", contentType: "text/roam+markdown", }); @@ -1015,7 +1015,7 @@ describe("materializeSharedNode", () => { it("brackets a media embed's URL before handing the markdown to Roam", async () => { const embed = `{{[[pdf]]: ${ASSET_URL}}}`; const { client } = clientWithFullContent({ - text: `# ${roamSharedNode.title}\n\n${embed}\n`, + text: `${embed}\n`, contentType: "text/roam+markdown", }); @@ -1034,7 +1034,7 @@ describe("materializeSharedNode", () => { it("brackets a media embed's URL when replacing an imported page too", async () => { const embed = `{{[[audio]]: ${ASSET_URL}}}`; const { client } = clientWithFullContent({ - text: `# ${roamSharedNode.title}\n\n${embed}\n`, + text: `${embed}\n`, contentType: "text/roam+markdown", }); mockedFindImportedNodeUidBySourceRid.mockResolvedValue(EXISTING_PAGE_UID); @@ -1052,9 +1052,9 @@ describe("materializeSharedNode", () => { }); }); - it("keeps a first line that does not match the shared title exactly", async () => { + it("normalizes CRLF in Roam body content", async () => { const { client } = clientWithFullContent({ - text: "# Some other heading\n\n- body", + text: "- first\r\n\r\n- second\r\n", contentType: "text/roam+markdown", }); @@ -1066,13 +1066,31 @@ describe("materializeSharedNode", () => { expect(result.success).toBe(true); expect(pageFromMarkdown).toHaveBeenCalledWith({ page: { title: roamSharedNode.title, uid: GENERATED_PAGE_UID }, - "markdown-string": "# Some other heading\n\n- body", + "markdown-string": "- first\n\n- second", }); }); - it("creates a title-only page when Roam full content is only the heading", async () => { + it("keeps a leading heading that repeats the title, since the body owns it", async () => { const { client } = clientWithFullContent({ - text: `# ${roamSharedNode.title}\n`, + text: `# ${roamSharedNode.title}\n\n- body`, + contentType: "text/roam+markdown", + }); + + const result = await materializeSharedNode({ + client, + sharedNode: roamSharedNode, + }); + + expect(result.success).toBe(true); + expect(pageFromMarkdown).toHaveBeenCalledWith({ + page: { title: roamSharedNode.title, uid: GENERATED_PAGE_UID }, + "markdown-string": `# ${roamSharedNode.title}\n\n- body`, + }); + }); + + it("creates a title-only page when the Roam body is empty", async () => { + const { client } = clientWithFullContent({ + text: "", contentType: "text/roam+markdown", }); @@ -1090,7 +1108,7 @@ describe("materializeSharedNode", () => { it("rejects Obsidian markdown on a Roam-origin node", async () => { const { client } = clientWithFullContent({ - text: `# ${roamSharedNode.title}\n\n- body`, + text: "- body", contentType: "text/obsidian+markdown", }); diff --git a/apps/roam/src/utils/__tests__/publishNodesToGroups.test.ts b/apps/roam/src/utils/__tests__/publishNodesToGroups.test.ts index d6a70d2544..669eb2fbad 100644 --- a/apps/roam/src/utils/__tests__/publishNodesToGroups.test.ts +++ b/apps/roam/src/utils/__tests__/publishNodesToGroups.test.ts @@ -103,7 +103,7 @@ const makeCrossAppNode = ({ direct: { localId: uid, value: title }, full: { localId: uid, - value: `# ${title}\n\nBody\n`, + value: `Body\n`, contentType: contentTypes.roamMarkdown, scale: "document", }, @@ -253,7 +253,7 @@ describe("publishNodesToGroups", () => { expect.objectContaining({ source_local_id: "node-1", variant: "full", - text: "# CLM - new claim\n\nBody\n", + text: "Body\n", content_type: contentTypes.roamMarkdown, }), ]); @@ -304,7 +304,7 @@ describe("publishNodesToGroups", () => { }), expect.objectContaining({ variant: "full", - text: "# CLM - updated title\n\nBody\n", + text: "Body\n", }), ]); expect(result.syncedNodeSchemaUids).toEqual([]); diff --git a/apps/roam/src/utils/convertRoamNodeToFullContent.example.ts b/apps/roam/src/utils/convertRoamNodeToFullContent.example.ts index 85547c1a70..78b73986d6 100644 --- a/apps/roam/src/utils/convertRoamNodeToFullContent.example.ts +++ b/apps/roam/src/utils/convertRoamNodeToFullContent.example.ts @@ -228,7 +228,7 @@ export const roamClaimFullMarkdownExample: { blocks, full: { contentType: contentTypes.markdown, - value: buildFullMarkdown({ title, blocks }), + value: buildFullMarkdown({ blocks }), authorId: "someone", }, }; diff --git a/apps/roam/src/utils/convertRoamNodeToFullContent.simple.example.ts b/apps/roam/src/utils/convertRoamNodeToFullContent.simple.example.ts index c174576626..deb1fa5fdf 100644 --- a/apps/roam/src/utils/convertRoamNodeToFullContent.simple.example.ts +++ b/apps/roam/src/utils/convertRoamNodeToFullContent.simple.example.ts @@ -36,6 +36,6 @@ export const roamClaimFullMarkdownSimpleExample: { blocks, full: { contentType: contentTypes.roamMarkdown, - value: buildFullMarkdown({ title, blocks }), + value: buildFullMarkdown({ blocks }), }, }; diff --git a/apps/roam/src/utils/materializeSharedNode.ts b/apps/roam/src/utils/materializeSharedNode.ts index f16448f35a..85011b99c2 100644 --- a/apps/roam/src/utils/materializeSharedNode.ts +++ b/apps/roam/src/utils/materializeSharedNode.ts @@ -1,7 +1,7 @@ import { contentTypes, + normalizeLineEndings, stripFrontmatter, - stripTitleHeading, trimBlankLines, } from "@repo/content-model"; import type { DGSupabaseClient } from "@repo/database/lib/client"; @@ -220,10 +220,12 @@ const fetchFullMarkdown = async ({ return { error: `Unsupported full content type "${data.content_type}" — expected "${expectedContentType}"`, }; + // Obsidian's `full` is the file bytes, so it opens with frontmatter that must + // not cross spaces. Roam's is the body alone. const withoutPreamble = - sharedNode.platform === "Roam" - ? stripTitleHeading({ markdown: data.text, title: sharedNode.title }) - : stripFrontmatter(data.text); + data.content_type === contentTypes.obsidianMarkdown + ? stripFrontmatter(data.text) + : normalizeLineEndings(data.text); const markdown = trimBlankLines(withoutPreamble); return { markdown: markdown.trim() ? markdown : "" }; }; diff --git a/apps/roam/src/utils/roamToCrossAppConverters.ts b/apps/roam/src/utils/roamToCrossAppConverters.ts index bbaeb61b88..5d91074690 100644 --- a/apps/roam/src/utils/roamToCrossAppConverters.ts +++ b/apps/roam/src/utils/roamToCrossAppConverters.ts @@ -34,12 +34,11 @@ const FULL_MARKDOWN_OPTS = { allNodes: [] as DiscourseNode[], }; +// `full` carries the page body alone; the title lives in the `direct` variant. export const buildFullMarkdown = ({ - title, blocks, viewType = "bullet", }: { - title: string; blocks: TreeNode[]; viewType?: ViewType; }): string => { @@ -50,7 +49,7 @@ export const buildFullMarkdown = ({ ) .join("\n") .trim(); - return body ? `# ${title}\n\n${body}\n` : `# ${title}\n`; + return body ? `${body}\n` : ""; }; const buildFullInlineContent = ({ @@ -64,7 +63,7 @@ const buildFullInlineContent = ({ const viewType = getPageViewType(title) || "bullet"; return { localId: uid, - value: buildFullMarkdown({ title, blocks, viewType }), + value: buildFullMarkdown({ blocks, viewType }), contentType: contentTypes.roamMarkdown, scale: "document", }; diff --git a/packages/content-model/src/__tests__/text.test.ts b/packages/content-model/src/__tests__/text.test.ts index 6d446abe33..ed05450ea0 100644 --- a/packages/content-model/src/__tests__/text.test.ts +++ b/packages/content-model/src/__tests__/text.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it } from "vitest"; import { normalizeLineEndings, stripFrontmatter, - stripTitleHeading, trimBlankLines, } from "../text/index.js"; @@ -24,36 +23,6 @@ describe("trimBlankLines", () => { }); }); -describe("stripTitleHeading", () => { - it("strips a title heading first line and following blank lines", () => { - expect( - stripTitleHeading({ markdown: "# Title\n\nbody", title: "Title" }), - ).toBe("body"); - }); - - it("leaves markdown whose first line differs from the title", () => { - expect( - stripTitleHeading({ markdown: "# Other\n\nbody", title: "Title" }), - ).toBe("# Other\n\nbody"); - }); - - it("requires an exact match, not a prefix", () => { - expect( - stripTitleHeading({ markdown: "# Title extra\nbody", title: "Title" }), - ).toBe("# Title extra\nbody"); - }); - - it("returns an empty string for heading-only markdown", () => { - expect(stripTitleHeading({ markdown: "# Title", title: "Title" })).toBe(""); - }); - - it("handles CRLF line endings", () => { - expect( - stripTitleHeading({ markdown: "# Title\r\nbody", title: "Title" }), - ).toBe("body"); - }); -}); - describe("stripFrontmatter", () => { it("removes a leading YAML frontmatter block and following blank lines", () => { expect( diff --git a/packages/content-model/src/text/index.ts b/packages/content-model/src/text/index.ts index 08e96d329e..50f1d7281e 100644 --- a/packages/content-model/src/text/index.ts +++ b/packages/content-model/src/text/index.ts @@ -4,22 +4,6 @@ export const normalizeLineEndings = (text: string): string => export const trimBlankLines = (text: string): string => text.replace(/^(?:[ \t]*\n)+/, "").replace(/(?:\n[ \t]*)+$/, ""); -export const stripTitleHeading = ({ - markdown, - title, -}: { - markdown: string; - title: string; -}): string => { - const normalized = normalizeLineEndings(markdown); - const newlineIndex = normalized.indexOf("\n"); - const firstLine = - newlineIndex === -1 ? normalized : normalized.slice(0, newlineIndex); - if (firstLine !== `# ${title}`) return normalized; - if (newlineIndex === -1) return ""; - return normalized.slice(newlineIndex + 1).replace(/^\n+/, ""); -}; - const FRONTMATTER_DELIMITER = "---"; export const stripFrontmatter = (markdown: string): string => { diff --git a/packages/database/schema.yaml b/packages/database/schema.yaml index b8d4d174e6..f83c088b7f 100644 --- a/packages/database/schema.yaml +++ b/packages/database/schema.yaml @@ -28,6 +28,8 @@ enums: direct: direct_and_children: direct_and_description: + full: + description: The whole source document, holding nothing that the direct variant already holds (such as the title) Validation: description: Whether a given value was given by a person, or suggested by an automated agent (and then possibly infirmed.) permissible_values: