From 8bec5b7a6a97e17398b445247d7a1ca22a2bad4a Mon Sep 17 00:00:00 2001 From: sid597 Date: Sun, 28 Jun 2026 22:31:53 +0530 Subject: [PATCH] [ENG-1858] Materialize Obsidian-origin shared nodes into Roam Add a Roam materializer that turns one Obsidian-origin CrossAppNode into a local page: parse content.full markdown into a Roam block tree, then create the page or update the one already imported from the same source (matched by sourceNodeRid, no duplicate), persisting imported source identity via the ENG-1856 helpers so it can be re-found and refreshed. - markdownToRoamBlocks: pure markdown -> InputTextNode[] (frontmatter strip, ATX headings -> Roam heading blocks, list nesting by indentation); unit-tested. - materializeSharedNode: create / update-by-RID / skip-if-unchanged, a title- collision guard that won't overwrite local pages, and a structured MaterializeResult for ENG-1859 to tally counts. Stacked on ENG-1856 (source-identity storage). Node-type reconciliation and the import command/UI are intentionally deferred (node-typing design question / ENG-1859). --- .../__tests__/markdownToRoamBlocks.test.ts | 81 +++++++++++++ apps/roam/src/utils/markdownToRoamBlocks.ts | 74 ++++++++++++ apps/roam/src/utils/materializeSharedNode.ts | 107 ++++++++++++++++++ 3 files changed, 262 insertions(+) create mode 100644 apps/roam/src/utils/__tests__/markdownToRoamBlocks.test.ts create mode 100644 apps/roam/src/utils/markdownToRoamBlocks.ts create mode 100644 apps/roam/src/utils/materializeSharedNode.ts diff --git a/apps/roam/src/utils/__tests__/markdownToRoamBlocks.test.ts b/apps/roam/src/utils/__tests__/markdownToRoamBlocks.test.ts new file mode 100644 index 000000000..6ba68c8b3 --- /dev/null +++ b/apps/roam/src/utils/__tests__/markdownToRoamBlocks.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect } from "vitest"; +import { markdownToRoamBlocks } from "../markdownToRoamBlocks"; + +describe("markdownToRoamBlocks", () => { + it("drops leading YAML frontmatter", () => { + expect( + markdownToRoamBlocks("---\nnodeTypeId: evd-1\n---\n\n# Title\n\nbody"), + ).toEqual([{ text: "Title", heading: 1 }, { text: "body" }]); + }); + + it("maps ATX headings to Roam heading blocks, clamped to 3", () => { + expect(markdownToRoamBlocks("# A\n## B\n### C\n#### D")).toEqual([ + { text: "A", heading: 1 }, + { text: "B", heading: 2 }, + { text: "C", heading: 3 }, + { text: "D", heading: 3 }, + ]); + }); + + it("does not treat #tag or empty headings as headings", () => { + expect(markdownToRoamBlocks("#tag is a ref\n#")).toEqual([ + { text: "#tag is a ref" }, + { text: "#" }, + ]); + }); + + it("nests list items by indentation", () => { + expect(markdownToRoamBlocks("- a\n - b\n - c\n- d")).toEqual([ + { text: "a", children: [{ text: "b" }, { text: "c" }] }, + { text: "d" }, + ]); + }); + + it("supports ordered lists and tab indentation", () => { + expect(markdownToRoamBlocks("1. one\n\t2. two")).toEqual([ + { text: "one", children: [{ text: "two" }] }, + ]); + }); + + it("treats each non-list line as its own top-level block", () => { + expect(markdownToRoamBlocks("para one\npara two")).toEqual([ + { text: "para one" }, + { text: "para two" }, + ]); + }); + + it("resets list nesting after a blank line or heading", () => { + expect(markdownToRoamBlocks("- a\n - b\n\n- c")).toEqual([ + { text: "a", children: [{ text: "b" }] }, + { text: "c" }, + ]); + }); + + it("returns [] for empty or frontmatter-only input", () => { + expect(markdownToRoamBlocks("")).toEqual([]); + expect(markdownToRoamBlocks("---\na: 1\n---\n")).toEqual([]); + }); + + it("parses the Obsidian-origin full markdown example", () => { + const md = + "---\nnodeTypeId: evd-7c1f9a2b\nnodeInstanceId: 0192f1a0\n---\n\n# REM sleep correlates with recall\n\nParticipants with more REM sleep showed better next-day recall.\n"; + expect(markdownToRoamBlocks(md)).toEqual([ + { text: "REM sleep correlates with recall", heading: 1 }, + { + text: "Participants with more REM sleep showed better next-day recall.", + }, + ]); + }); + + it("parses Roam-origin full markdown (H1 + paragraph + bullet)", () => { + const md = + "# Sleep improves memory consolidation\n\nMultiple studies show that sleep after learning strengthens memory traces.\n\n- Supported by [[EVD]] - Rasch & Born 2013\n"; + expect(markdownToRoamBlocks(md)).toEqual([ + { text: "Sleep improves memory consolidation", heading: 1 }, + { + text: "Multiple studies show that sleep after learning strengthens memory traces.", + }, + { text: "Supported by [[EVD]] - Rasch & Born 2013" }, + ]); + }); +}); diff --git a/apps/roam/src/utils/markdownToRoamBlocks.ts b/apps/roam/src/utils/markdownToRoamBlocks.ts new file mode 100644 index 000000000..025df121a --- /dev/null +++ b/apps/roam/src/utils/markdownToRoamBlocks.ts @@ -0,0 +1,74 @@ +import type { InputTextNode } from "roamjs-components/types"; + +const FRONTMATTER_RE = /^---\r?\n[\s\S]*?\r?\n---[ \t]*(?:\r?\n|$)/; +const HEADING_RE = /^(#{1,6})\s+(.*\S)\s*$/; +const LIST_ITEM_RE = /^(\s*)(?:[-*+]|\d+[.)])\s+(.*)$/; + +const stripFrontmatter = (markdown: string): string => + markdown.replace(FRONTMATTER_RE, ""); + +type ListFrame = { node: InputTextNode; indent: number }; + +/** + * Convert a markdown document into a Roam block tree (`InputTextNode[]`) — the + * shape `createPage`/`createBlock` consume. Used when materializing an + * Obsidian-origin shared node's `content.full` markdown into a local Roam page. + * + * MVP0 mapping: leading YAML frontmatter is dropped; ATX headings (`#`..`######`) + * become Roam heading blocks (levels clamped to Roam's 1–3); list items nest by + * indentation; every other non-blank line becomes its own top-level block. Inline + * markup, wikilinks, and block refs pass through verbatim for Roam to render. + */ +export const markdownToRoamBlocks = (markdown: string): InputTextNode[] => { + const roots: InputTextNode[] = []; + const listStack: ListFrame[] = []; + + const appendBlock = ( + node: InputTextNode, + listIndent: number | null, + ): void => { + if (listIndent === null) { + roots.push(node); + listStack.length = 0; + return; + } + while ( + listStack.length && + listStack[listStack.length - 1].indent >= listIndent + ) { + listStack.pop(); + } + const parent = listStack[listStack.length - 1]; + if (parent) { + (parent.node.children ??= []).push(node); + } else { + roots.push(node); + } + listStack.push({ node, indent: listIndent }); + }; + + for (const rawLine of stripFrontmatter(markdown).split("\n")) { + const line = rawLine.replace(/\s+$/, ""); + if (!line.trim()) continue; + + const heading = HEADING_RE.exec(line); + if (heading) { + appendBlock( + { text: heading[2], heading: Math.min(heading[1].length, 3) }, + null, + ); + continue; + } + + const listItem = LIST_ITEM_RE.exec(line); + if (listItem) { + const indent = listItem[1].replace(/\t/g, " ").length; + appendBlock({ text: listItem[2] }, indent); + continue; + } + + appendBlock({ text: line.trim() }, null); + } + + return roots; +}; diff --git a/apps/roam/src/utils/materializeSharedNode.ts b/apps/roam/src/utils/materializeSharedNode.ts new file mode 100644 index 000000000..16bc5c188 --- /dev/null +++ b/apps/roam/src/utils/materializeSharedNode.ts @@ -0,0 +1,107 @@ +import createPage from "roamjs-components/writes/createPage"; +import createBlock from "roamjs-components/writes/createBlock"; +import deleteBlock from "roamjs-components/writes/deleteBlock"; +import getFullTreeByParentUid from "roamjs-components/queries/getFullTreeByParentUid"; +import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle"; +import getPageTitleByPageUid from "roamjs-components/queries/getPageTitleByPageUid"; +import type { InputTextNode } from "roamjs-components/types"; +import type { CrossAppNode } from "@repo/database/crossAppNodeContract"; +import { + findImportedNodeUidByRid, + readImportedSourceIdentity, + toImportedSourceIdentity, + writeImportedSourceIdentity, +} from "./importedSourceIdentity"; +import { markdownToRoamBlocks } from "./markdownToRoamBlocks"; + +/** + * Outcome of materializing one shared node, so the import action (ENG-1859) can + * tally counts. `skipped` means the page was already imported and unchanged. + */ +export type MaterializeResult = + | { status: "created" | "updated" | "skipped"; pageUid: string } + | { status: "failed"; error: string }; + +/** Overwrite a page's body: clear its existing blocks, then write the new tree. */ +const replacePageBody = async ( + pageUid: string, + blocks: InputTextNode[], +): Promise => { + const existing = getFullTreeByParentUid(pageUid).children ?? []; + await Promise.all(existing.map((child) => deleteBlock(child.uid))); + await Promise.all( + blocks.map((node, order) => + createBlock({ parentUid: pageUid, node, order }), + ), + ); +}; + +/** + * Materialize one Obsidian-origin shared node into the local Roam graph: create a + * page from its `content.full` markdown, or update the page already imported from + * the same source (matched by `sourceNodeRid`) without creating a duplicate. The + * imported source identity is (re)written to the page's block props so the node + * can be re-found and refreshed. Pure markdown parsing lives in + * `markdownToRoamBlocks`; identity persistence/lookup in `importedSourceIdentity`. + * + * MVP0 keeps the source's (already typed) title verbatim and does not yet + * reconcile `node.nodeType` against local Roam node-type definitions, so an + * imported node is recognized as a discourse node only when its title already + * matches a local node format; active type mapping/creation is deferred. + * + * Returns a structured outcome rather than throwing. An already-imported node's + * stored identity is never dropped: on update the body is replaced first and the + * identity (carrying the new `sourceModifiedAt`) is written last, so a mid-update + * failure leaves the prior identity intact and the node re-importable. + */ +export const materializeSharedNode = async ( + node: CrossAppNode, +): Promise => { + try { + const title = node.content.direct.value; + if (!title.trim()) { + return { status: "failed", error: "Shared node has no title" }; + } + + const importedUid = await findImportedNodeUidByRid(node.sourceNodeRid); + if ( + importedUid && + readImportedSourceIdentity(importedUid)?.sourceModifiedAt === + node.sourceModifiedAt + ) { + return { status: "skipped", pageUid: importedUid }; + } + + const identity = toImportedSourceIdentity(node); + const blocks = markdownToRoamBlocks(node.content.full.value); + + if (importedUid) { + const currentTitle = getPageTitleByPageUid(importedUid); + if (currentTitle && currentTitle !== title) { + await window.roamAlphaAPI.updatePage({ + page: { uid: importedUid, title }, + }); + } + await replacePageBody(importedUid, blocks); + writeImportedSourceIdentity(importedUid, identity); + return { status: "updated", pageUid: importedUid }; + } + + const collidingUid = getPageUidByPageTitle(title); + if (collidingUid) { + return { + status: "failed", + error: `A page titled "${title}" already exists locally and was not imported from this source; skipping to avoid overwriting local content.`, + }; + } + + const pageUid = await createPage({ title, tree: blocks }); + writeImportedSourceIdentity(pageUid, identity); + return { status: "created", pageUid }; + } catch (error) { + return { + status: "failed", + error: error instanceof Error ? error.message : String(error), + }; + } +};