[ENG-1858] Materialize Obsidian-origin markdown into Roam

apps/roam/src/utils/__tests__/markdownToRoamBlocks.test.ts

@@ -0,0 +1,81 @@
+import { describe, it, expect } from "vitest";
+import { markdownToRoamBlocks } from "../markdownToRoamBlocks";
+
+describe("markdownToRoamBlocks", () => {
+  it("drops leading YAML frontmatter", () => {
+    expect(
+      markdownToRoamBlocks("---\nnodeTypeId: evd-1\n---\n\n# Title\n\nbody"),
+    ).toEqual([{ text: "Title", heading: 1 }, { text: "body" }]);
+  });
+
+  it("maps ATX headings to Roam heading blocks, clamped to 3", () => {
+    expect(markdownToRoamBlocks("# A\n## B\n### C\n#### D")).toEqual([
+      { text: "A", heading: 1 },
+      { text: "B", heading: 2 },
+      { text: "C", heading: 3 },
+      { text: "D", heading: 3 },
+    ]);
+  });
+
+  it("does not treat #tag or empty headings as headings", () => {
+    expect(markdownToRoamBlocks("#tag is a ref\n#")).toEqual([
+      { text: "#tag is a ref" },
+      { text: "#" },
+    ]);
+  });
+
+  it("nests list items by indentation", () => {
+    expect(markdownToRoamBlocks("- a\n  - b\n  - c\n- d")).toEqual([
+      { text: "a", children: [{ text: "b" }, { text: "c" }] },
+      { text: "d" },
+    ]);
+  });
+
+  it("supports ordered lists and tab indentation", () => {
+    expect(markdownToRoamBlocks("1. one\n\t2. two")).toEqual([
+      { text: "one", children: [{ text: "two" }] },
+    ]);
+  });
+
+  it("treats each non-list line as its own top-level block", () => {
+    expect(markdownToRoamBlocks("para one\npara two")).toEqual([
+      { text: "para one" },
+      { text: "para two" },
+    ]);
+  });
+
+  it("resets list nesting after a blank line or heading", () => {
+    expect(markdownToRoamBlocks("- a\n  - b\n\n- c")).toEqual([
+      { text: "a", children: [{ text: "b" }] },
+      { text: "c" },
+    ]);
+  });
+
+  it("returns [] for empty or frontmatter-only input", () => {
+    expect(markdownToRoamBlocks("")).toEqual([]);
+    expect(markdownToRoamBlocks("---\na: 1\n---\n")).toEqual([]);
+  });
+
+  it("parses the Obsidian-origin full markdown example", () => {
+    const md =
+      "---\nnodeTypeId: evd-7c1f9a2b\nnodeInstanceId: 0192f1a0\n---\n\n# REM sleep correlates with recall\n\nParticipants with more REM sleep showed better next-day recall.\n";
+    expect(markdownToRoamBlocks(md)).toEqual([
+      { text: "REM sleep correlates with recall", heading: 1 },
+      {
+        text: "Participants with more REM sleep showed better next-day recall.",
+      },
+    ]);
+  });
+
+  it("parses Roam-origin full markdown (H1 + paragraph + bullet)", () => {
+    const md =
+      "# Sleep improves memory consolidation\n\nMultiple studies show that sleep after learning strengthens memory traces.\n\n- Supported by [[EVD]] - Rasch & Born 2013\n";
+    expect(markdownToRoamBlocks(md)).toEqual([
+      { text: "Sleep improves memory consolidation", heading: 1 },
+      {
+        text: "Multiple studies show that sleep after learning strengthens memory traces.",
+      },
+      { text: "Supported by [[EVD]] - Rasch & Born 2013" },
+    ]);
+  });
+});

apps/roam/src/utils/markdownToRoamBlocks.ts

@@ -0,0 +1,74 @@
+import type { InputTextNode } from "roamjs-components/types";
+
+const FRONTMATTER_RE = /^---\r?\n[\s\S]*?\r?\n---[ \t]*(?:\r?\n|$)/;
+const HEADING_RE = /^(#{1,6})\s+(.*\S)\s*$/;
+const LIST_ITEM_RE = /^(\s*)(?:[-*+]|\d+[.)])\s+(.*)$/;
+
+const stripFrontmatter = (markdown: string): string =>
+  markdown.replace(FRONTMATTER_RE, "");
+
+type ListFrame = { node: InputTextNode; indent: number };
+
+/**
+ * Convert a markdown document into a Roam block tree (`InputTextNode[]`) — the
+ * shape `createPage`/`createBlock` consume. Used when materializing an
+ * Obsidian-origin shared node's `content.full` markdown into a local Roam page.
+ *
+ * MVP0 mapping: leading YAML frontmatter is dropped; ATX headings (`#`..`######`)
+ * become Roam heading blocks (levels clamped to Roam's 1–3); list items nest by
+ * indentation; every other non-blank line becomes its own top-level block. Inline
+ * markup, wikilinks, and block refs pass through verbatim for Roam to render.
+ */
+export const markdownToRoamBlocks = (markdown: string): InputTextNode[] => {
+  const roots: InputTextNode[] = [];
+  const listStack: ListFrame[] = [];
+
+  const appendBlock = (
+    node: InputTextNode,
+    listIndent: number | null,
+  ): void => {
+    if (listIndent === null) {
+      roots.push(node);
+      listStack.length = 0;
+      return;
+    }
+    while (
+      listStack.length &&
+      listStack[listStack.length - 1].indent >= listIndent
+    ) {
+      listStack.pop();
+    }
+    const parent = listStack[listStack.length - 1];
+    if (parent) {
+      (parent.node.children ??= []).push(node);
+    } else {
+      roots.push(node);
+    }
+    listStack.push({ node, indent: listIndent });
+  };
+
+  for (const rawLine of stripFrontmatter(markdown).split("\n")) {
+    const line = rawLine.replace(/\s+$/, "");
+    if (!line.trim()) continue;
+
+    const heading = HEADING_RE.exec(line);
+    if (heading) {
+      appendBlock(
+        { text: heading[2], heading: Math.min(heading[1].length, 3) },
+        null,
+      );
+      continue;
+    }
+
+    const listItem = LIST_ITEM_RE.exec(line);
+    if (listItem) {
+      const indent = listItem[1].replace(/\t/g, "  ").length;
+      appendBlock({ text: listItem[2] }, indent);
+      continue;
+    }
+
+    appendBlock({ text: line.trim() }, null);
+  }
+
+  return roots;
+};

apps/roam/src/utils/materializeSharedNode.ts

@@ -0,0 +1,107 @@
+import createPage from "roamjs-components/writes/createPage";
+import createBlock from "roamjs-components/writes/createBlock";
+import deleteBlock from "roamjs-components/writes/deleteBlock";
+import getFullTreeByParentUid from "roamjs-components/queries/getFullTreeByParentUid";
+import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle";
+import getPageTitleByPageUid from "roamjs-components/queries/getPageTitleByPageUid";
+import type { InputTextNode } from "roamjs-components/types";
+import type { CrossAppNode } from "@repo/database/crossAppNodeContract";
+import {
+  findImportedNodeUidByRid,
+  readImportedSourceIdentity,
+  toImportedSourceIdentity,
+  writeImportedSourceIdentity,
+} from "./importedSourceIdentity";
+import { markdownToRoamBlocks } from "./markdownToRoamBlocks";
+
+/**
+ * Outcome of materializing one shared node, so the import action (ENG-1859) can
+ * tally counts. `skipped` means the page was already imported and unchanged.
+ */
+export type MaterializeResult =
+  | { status: "created" | "updated" | "skipped"; pageUid: string }
+  | { status: "failed"; error: string };
+
+/** Overwrite a page's body: clear its existing blocks, then write the new tree. */
+const replacePageBody = async (
+  pageUid: string,
+  blocks: InputTextNode[],
+): Promise<void> => {
+  const existing = getFullTreeByParentUid(pageUid).children ?? [];
+  await Promise.all(existing.map((child) => deleteBlock(child.uid)));
+  await Promise.all(
+    blocks.map((node, order) =>
+      createBlock({ parentUid: pageUid, node, order }),
+    ),
+  );
+};
+
+/**
+ * Materialize one Obsidian-origin shared node into the local Roam graph: create a
+ * page from its `content.full` markdown, or update the page already imported from
+ * the same source (matched by `sourceNodeRid`) without creating a duplicate. The
+ * imported source identity is (re)written to the page's block props so the node
+ * can be re-found and refreshed. Pure markdown parsing lives in
+ * `markdownToRoamBlocks`; identity persistence/lookup in `importedSourceIdentity`.
+ *
+ * MVP0 keeps the source's (already typed) title verbatim and does not yet
+ * reconcile `node.nodeType` against local Roam node-type definitions, so an
+ * imported node is recognized as a discourse node only when its title already
+ * matches a local node format; active type mapping/creation is deferred.
+ *
+ * Returns a structured outcome rather than throwing. An already-imported node's
+ * stored identity is never dropped: on update the body is replaced first and the
+ * identity (carrying the new `sourceModifiedAt`) is written last, so a mid-update
+ * failure leaves the prior identity intact and the node re-importable.
+ */
+export const materializeSharedNode = async (
+  node: CrossAppNode,
+): Promise<MaterializeResult> => {
+  try {
+    const title = node.content.direct.value;
+    if (!title.trim()) {
+      return { status: "failed", error: "Shared node has no title" };
+    }
+
+    const importedUid = await findImportedNodeUidByRid(node.sourceNodeRid);
+    if (
+      importedUid &&
+      readImportedSourceIdentity(importedUid)?.sourceModifiedAt ===
+        node.sourceModifiedAt
+    ) {
+      return { status: "skipped", pageUid: importedUid };
+    }
+
+    const identity = toImportedSourceIdentity(node);
+    const blocks = markdownToRoamBlocks(node.content.full.value);
+
+    if (importedUid) {
+      const currentTitle = getPageTitleByPageUid(importedUid);
+      if (currentTitle && currentTitle !== title) {
+        await window.roamAlphaAPI.updatePage({
+          page: { uid: importedUid, title },
+        });
+      }
+      await replacePageBody(importedUid, blocks);
+      writeImportedSourceIdentity(importedUid, identity);
+      return { status: "updated", pageUid: importedUid };
+    }
+
+    const collidingUid = getPageUidByPageTitle(title);
+    if (collidingUid) {
+      return {
+        status: "failed",
+        error: `A page titled "${title}" already exists locally and was not imported from this source; skipping to avoid overwriting local content.`,
+      };
+    }
+
+    const pageUid = await createPage({ title, tree: blocks });
+    writeImportedSourceIdentity(pageUid, identity);
+    return { status: "created", pageUid };
+  } catch (error) {
+    return {
+      status: "failed",
+      error: error instanceof Error ? error.message : String(error),
+    };
+  }
+};