From 5e21302e238fe2ef772585413dd8ef603151a193 Mon Sep 17 00:00:00 2001 From: Michael Gartner Date: Sun, 5 Jul 2026 23:17:32 -0600 Subject: [PATCH 1/3] Enhance content management by adding support for content types and variants - Introduced a new scenario in the feature file to test coexistence of different content types. - Updated step definitions to include checks for content rows based on variant and content type. - Modified database types to include `content_type` in relevant tables and relationships. - Adjusted SQL schemas to enforce content type constraints and ensure proper indexing. - Backfilled existing rows with default content types to maintain data integrity. --- packages/database/features/addContent.feature | 49 ++++ .../features/step-definitions/stepdefs.ts | 25 ++ packages/database/src/dbTypes.ts | 31 ++- ...260704120000_update_content_uniqueness.sql | 239 ++++++++++++++++++ packages/database/supabase/schemas/assets.sql | 5 +- .../database/supabase/schemas/content.sql | 14 +- .../database/supabase/schemas/embedding.sql | 3 +- 7 files changed, 351 insertions(+), 15 deletions(-) create mode 100644 packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql diff --git a/packages/database/features/addContent.feature b/packages/database/features/addContent.feature index 9148b6328..35e8b237a 100644 --- a/packages/database/features/addContent.feature +++ b/packages/database/features/addContent.feature @@ -133,3 +133,52 @@ Feature: Content access And a user logged in space s1 should see 3 Content in the database And a user logged in space s1 should see 1 ContentEmbedding_openai_text_embedding_3_small_1536 in the database And a user logged in space s1 should see 1 Document in the database + + Scenario: Full content representations with different content types coexist + When user user1 upserts these documents to space s1: + """json + [ + { + "source_local_id": "page1_uid", + "created": "2000/01/01", + "last_modified": "2001/01/02", + "author_local_id": "user1", + "content_type": "text/markdown" + } + ] + """ + And user user1 upserts this content to space s1: + """json + [ + { + "author_local_id": "user1", + "document_local_id": "page1_uid", + "source_local_id": "page1_uid", + "variant": "full", + "scale": "document", + "created": "2000/01/01", + "last_modified": "2001/01/02", + "text": "# Markdown", + "content_type": "text/markdown" + } + ] + """ + And user user1 upserts this content to space s1: + """json + [ + { + "author_local_id": "user1", + "document_local_id": "page1_uid", + "source_local_id": "page1_uid", + "variant": "full", + "scale": "document", + "created": "2000/01/01", + "last_modified": "2001/01/02", + "text": "{\"type\":\"root\",\"children\":[]}", + "content_type": "application/vnd.discourse-graph.atjson+json; version=1" + } + ] + """ + Then a user logged in space s1 should see 2 Content in the database + And a user logged in space s1 should see 1 content rows with variant "full" and content type "text/markdown" + And a user logged in space s1 should see 1 content rows with variant "full" and content type "application/vnd.discourse-graph.atjson+json; version=1" diff --git a/packages/database/features/step-definitions/stepdefs.ts b/packages/database/features/step-definitions/stepdefs.ts index 676ff52aa..36aad239a 100644 --- a/packages/database/features/step-definitions/stepdefs.ts +++ b/packages/database/features/step-definitions/stepdefs.ts @@ -21,6 +21,7 @@ import { } from "@repo/database/lib/contextFunctions"; type Platform = Enums<"Platform">; +type ContentVariant = Enums<"ContentVariant">; type TableName = keyof Database["public"]["Tables"]; type LocalRefsType = Record; const PLATFORMS: readonly Platform[] = Constants.public.Enums.Platform; @@ -307,6 +308,30 @@ Then( }, ); +Then( + "a user logged in space {word} should see {int} content rows with variant {string} and content type {string}", + async ( + ...[spaceName, expectedCount, variant, contentType]: [ + string, + number, + ContentVariant, + string, + ] + ) => { + const localRefs = (world.localRefs || {}) as LocalRefsType; + const spaceId = localRefs[spaceName]; + if (typeof spaceId !== "number") assert.fail("spaceId not a number"); + const client = await getLoggedinDatabase(spaceId); + const response = await client + .from("my_contents") + .select("*", { count: "exact", head: true }) + .eq("variant", variant) + .eq("content_type", contentType); + assert.equal(response.error, null); + assert.equal(response.count, expectedCount); + }, +); + // invoke the upsert_accounts_in_space function, expects json Given( "user {word} upserts these accounts to space {word}:", diff --git a/packages/database/src/dbTypes.ts b/packages/database/src/dbTypes.ts index eff6b23a5..a94c55b25 100644 --- a/packages/database/src/dbTypes.ts +++ b/packages/database/src/dbTypes.ts @@ -587,6 +587,7 @@ export type Database = { } FileReference: { Row: { + content_type: string created: string filehash: string filepath: string @@ -596,6 +597,7 @@ export type Database = { variant: Database["public"]["Enums"]["ContentVariant"] | null } Insert: { + content_type?: string created: string filehash: string filepath: string @@ -605,6 +607,7 @@ export type Database = { variant?: Database["public"]["Enums"]["ContentVariant"] | null } Update: { + content_type?: string created?: string filehash?: string filepath?: string @@ -616,24 +619,39 @@ export type Database = { Relationships: [ { foreignKeyName: "FileReference_content_fkey" - columns: ["space_id", "source_local_id", "variant"] + columns: ["space_id", "source_local_id", "variant", "content_type"] isOneToOne: false referencedRelation: "Content" - referencedColumns: ["space_id", "source_local_id", "variant"] + referencedColumns: [ + "space_id", + "source_local_id", + "variant", + "content_type", + ] }, { foreignKeyName: "FileReference_content_fkey" - columns: ["space_id", "source_local_id", "variant"] + columns: ["space_id", "source_local_id", "variant", "content_type"] isOneToOne: false referencedRelation: "my_contents" - referencedColumns: ["space_id", "source_local_id", "variant"] + referencedColumns: [ + "space_id", + "source_local_id", + "variant", + "content_type", + ] }, { foreignKeyName: "FileReference_content_fkey" - columns: ["space_id", "source_local_id", "variant"] + columns: ["space_id", "source_local_id", "variant", "content_type"] isOneToOne: false referencedRelation: "my_contents_with_embedding_openai_text_embedding_3_small_1536" - referencedColumns: ["space_id", "source_local_id", "variant"] + referencedColumns: [ + "space_id", + "source_local_id", + "variant", + "content_type", + ] }, ] } @@ -1164,6 +1182,7 @@ export type Database = { text: string | null variant: Database["public"]["Enums"]["ContentVariant"] | null vector: string | null + content_type: string | null } Relationships: [ { diff --git a/packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql b/packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql new file mode 100644 index 000000000..b459a6f90 --- /dev/null +++ b/packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql @@ -0,0 +1,239 @@ +ALTER TABLE ONLY public."FileReference" +DROP CONSTRAINT IF EXISTS "FileReference_content_fkey"; + +ALTER TABLE public."FileReference" +ADD COLUMN IF NOT EXISTS content_type character varying NOT NULL DEFAULT 'text/obsidian+markdown'; + +UPDATE public."FileReference" AS fr +SET content_type = ct.content_type +FROM public."Content" AS ct +WHERE fr.space_id = ct.space_id + AND fr.source_local_id = ct.source_local_id + AND fr.variant = ct.variant; + +DROP INDEX IF EXISTS public.content_space_local_id_variant_idx; + +CREATE UNIQUE INDEX IF NOT EXISTS content_space_local_id_variant_content_type_idx ON public."Content" USING btree ( + space_id, source_local_id, variant, content_type +) NULLS DISTINCT; + +ALTER TABLE ONLY public."FileReference" +ADD CONSTRAINT "FileReference_content_fkey" FOREIGN KEY ( + space_id, source_local_id, variant, content_type +) REFERENCES public."Content" (space_id, source_local_id, variant, content_type) ON DELETE CASCADE; + +CREATE OR REPLACE FUNCTION public._local_content_to_db_content(data public.content_local_input) +RETURNS public."Content" STABLE +SET search_path = '' +LANGUAGE plpgsql AS $$ +DECLARE + content public."Content"%ROWTYPE; + reference_content JSONB := jsonb_build_object(); + key varchar; + value JSONB; + ref_single_val BIGINT; + ref_array_val BIGINT[]; +BEGIN + content := jsonb_populate_record(NULL::public."Content", to_jsonb(data)); + IF data.document_local_id IS NOT NULL THEN + SELECT id FROM public."Document" + WHERE source_local_id = data.document_local_id INTO content.document_id; + END IF; + IF data.creator_local_id IS NOT NULL THEN + SELECT id FROM public."PlatformAccount" + WHERE account_local_id = data.creator_local_id INTO content.creator_id; + ELSIF account_local_id(creator_inline(data)) IS NOT NULL THEN + SELECT id FROM public."PlatformAccount" + WHERE account_local_id = account_local_id(creator_inline(data)) INTO content.creator_id; + END IF; + IF data.author_local_id IS NOT NULL THEN + SELECT id FROM public."PlatformAccount" + WHERE account_local_id = data.author_local_id INTO content.author_id; + ELSIF account_local_id(author_inline(data)) IS NOT NULL THEN + SELECT id FROM public."PlatformAccount" + WHERE account_local_id = account_local_id(author_inline(data)) INTO content.author_id; + END IF; + IF data.part_of_local_id IS NOT NULL THEN + SELECT id FROM public."Content" + WHERE source_local_id = data.part_of_local_id INTO content.part_of_id; + END IF; + IF data.space_url IS NOT NULL THEN + SELECT id FROM public."Space" + WHERE url = data.space_url INTO content.space_id; + END IF; + -- now avoid null defaults + IF content.metadata IS NULL then + content.metadata := '{}'; + END IF; + IF content.content_type IS NULL THEN + content.content_type := 'text/plain'; + END IF; + RETURN content; +END; +$$; + +COMMENT ON FUNCTION public._local_content_to_db_content IS 'utility function so we have the option to use platform identifiers for content upsert'; + +CREATE OR REPLACE FUNCTION public.upsert_content(v_space_id bigint, data jsonb, v_creator_id BIGINT, content_as_document boolean DEFAULT TRUE) +RETURNS SETOF BIGINT +SET search_path = '' +LANGUAGE plpgsql +AS $$ +DECLARE + v_platform public."Platform"; + db_document public."Document"%ROWTYPE; + document_id BIGINT; + local_content public.content_local_input; + db_content public."Content"%ROWTYPE; + content_row JSONB; + upsert_id BIGINT; +BEGIN + SELECT platform INTO STRICT v_platform FROM public."Space" WHERE id=v_space_id; + FOR content_row IN SELECT * FROM jsonb_array_elements(data) + LOOP + local_content := jsonb_populate_record(NULL::public.content_local_input, content_row); + local_content.space_id := v_space_id; + db_content := public._local_content_to_db_content(local_content); + IF account_local_id(author_inline(local_content)) IS NOT NULL THEN + SELECT public.create_account_in_space( + v_space_id, + account_local_id(author_inline(local_content)), + name(author_inline(local_content)) + ) INTO STRICT upsert_id; + db_content.author_id := upsert_id; + END IF; + IF account_local_id(creator_inline(local_content)) IS NOT NULL THEN + SELECT public.create_account_in_space( + v_space_id, + account_local_id(creator_inline(local_content)), + name(creator_inline(local_content)) + ) INTO STRICT upsert_id; + db_content.creator_id := upsert_id; + END IF; + IF content_as_document THEN + db_content.scale = 'document'; + END IF; + IF content_as_document AND document_id(db_content) IS NULL AND source_local_id(document_inline(local_content)) IS NULL THEN + local_content.document_inline.space_id := v_space_id; + local_content.document_inline.source_local_id := db_content.source_local_id; + local_content.document_inline.last_modified := db_content.last_modified; + local_content.document_inline.created := db_content.created; + local_content.document_inline.author_id := db_content.author_id; + END IF; + IF source_local_id(document_inline(local_content)) IS NOT NULL THEN + IF content_type(document_inline(local_content)) IS NULL THEN + local_content.document_inline.content_type := CASE + WHEN v_platform='Roam' THEN 'text/roam+markdown' + WHEN v_platform='Obsidian' THEN 'text/obsidian+markdown' + ELSE 'text/plain' END; + END IF; + db_document := public._local_document_to_db_document(document_inline(local_content)); + IF (db_document.author_id IS NULL AND author_inline(local_content) IS NOT NULL) THEN + db_document.author_id := upsert_account_in_space(v_space_id, author_inline(local_content)); + END IF; + INSERT INTO public."Document" ( + space_id, + source_local_id, + url, + created, + metadata, + last_modified, + author_id, + contents, + content_type + ) VALUES ( + COALESCE(db_document.space_id, v_space_id), + db_document.source_local_id, + db_document.url, + db_document.created, + COALESCE(db_document.metadata, '{}'::jsonb), + db_document.last_modified, + db_document.author_id, + db_document.contents, + db_document.content_type + ) + ON CONFLICT (space_id, source_local_id) DO UPDATE SET + url = COALESCE(db_document.url, EXCLUDED.url), + created = COALESCE(db_document.created, EXCLUDED.created), + metadata = COALESCE(db_document.metadata, EXCLUDED.metadata), + last_modified = COALESCE(db_document.last_modified, EXCLUDED.last_modified), + author_id = COALESCE(db_document.author_id, EXCLUDED.author_id), + contents = COALESCE(db_document.contents, EXCLUDED.contents), + content_type = COALESCE(db_document.content_type, EXCLUDED.content_type) + RETURNING id INTO STRICT document_id; + db_content.document_id := document_id; + END IF; + INSERT INTO public."Content" ( + document_id, + source_local_id, + variant, + author_id, + creator_id, + created, + text, + metadata, + scale, + space_id, + last_modified, + part_of_id, + content_type + ) VALUES ( + db_content.document_id, + db_content.source_local_id, + COALESCE(db_content.variant, 'direct'::public."ContentVariant"), + db_content.author_id, + db_content.creator_id, + db_content.created, + db_content.text, + COALESCE(db_content.metadata, '{}'::jsonb), + db_content.scale, + db_content.space_id, + db_content.last_modified, + db_content.part_of_id, + db_content.content_type + ) + ON CONFLICT (space_id, source_local_id, variant, content_type) DO UPDATE SET + document_id = COALESCE(db_content.document_id, EXCLUDED.document_id), + author_id = COALESCE(db_content.author_id, EXCLUDED.author_id), + creator_id = COALESCE(db_content.creator_id, EXCLUDED.creator_id), + created = COALESCE(db_content.created, EXCLUDED.created), + text = COALESCE(db_content.text, EXCLUDED.text), + metadata = COALESCE(db_content.metadata, EXCLUDED.metadata), + scale = COALESCE(db_content.scale, EXCLUDED.scale), + last_modified = COALESCE(db_content.last_modified, EXCLUDED.last_modified), + part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id) + RETURNING id INTO STRICT upsert_id; + IF model(embedding_inline(local_content)) IS NOT NULL THEN + PERFORM public.upsert_content_embedding(upsert_id, model(embedding_inline(local_content)), vector(embedding_inline(local_content))); + END IF; + RETURN NEXT upsert_id; + END LOOP; +END; +$$; + +COMMENT ON FUNCTION public.upsert_content IS 'batch content upsert'; + +CREATE OR REPLACE VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536 AS +SELECT +ct.id, +ct.document_id, +ct.source_local_id, +ct.variant, +ct.author_id, +ct.creator_id, +ct.created, +ct.text, +ct.metadata, +ct.scale, +ct.space_id, +ct.last_modified, +ct.part_of_id, +emb.model, +emb.vector, +ct.content_type +FROM public."Content" AS ct +JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id) +LEFT OUTER JOIN public.my_accessible_resources () AS ra USING (space_id, source_local_id) +WHERE (ct.space_id = any (public.my_space_ids ('reader')) +OR (ct.space_id = any (public.my_space_ids ('partial')) AND ra.space_id IS NOT NULL)) +AND NOT emb.obsolete; diff --git a/packages/database/supabase/schemas/assets.sql b/packages/database/supabase/schemas/assets.sql index 001fb2977..2e5003be3 100644 --- a/packages/database/supabase/schemas/assets.sql +++ b/packages/database/supabase/schemas/assets.sql @@ -5,6 +5,7 @@ CREATE TABLE IF NOT EXISTS public."FileReference" ( filehash character varying NOT NULL, -- or binary? "created" timestamp without time zone NOT NULL, last_modified timestamp without time zone NOT NULL, + content_type character varying NOT NULL DEFAULT 'text/obsidian+markdown', -- not allowed virtual with user types variant public."ContentVariant" GENERATED ALWAYS AS ('full') STORED ); @@ -13,8 +14,8 @@ ADD CONSTRAINT "FileReference_pkey" PRIMARY KEY (source_local_id, space_id, file ALTER TABLE ONLY public."FileReference" ADD CONSTRAINT "FileReference_content_fkey" FOREIGN KEY ( - space_id, source_local_id, variant -) REFERENCES public."Content" (space_id, source_local_id, variant) ON DELETE CASCADE; + space_id, source_local_id, variant, content_type +) REFERENCES public."Content" (space_id, source_local_id, variant, content_type) ON DELETE CASCADE; -- note the absence of on update ; the generated column forbids cascade, so it will error -- However, update on those columns should never happen. diff --git a/packages/database/supabase/schemas/content.sql b/packages/database/supabase/schemas/content.sql index 54f2a3bc9..a41846c6e 100644 --- a/packages/database/supabase/schemas/content.sql +++ b/packages/database/supabase/schemas/content.sql @@ -131,8 +131,8 @@ CREATE INDEX "Content_part_of" ON public."Content" USING btree ( CREATE INDEX "Content_space" ON public."Content" USING btree (space_id); -CREATE UNIQUE INDEX content_space_local_id_variant_idx ON public."Content" USING btree ( - space_id, source_local_id, variant +CREATE UNIQUE INDEX content_space_local_id_variant_content_type_idx ON public."Content" USING btree ( + space_id, source_local_id, variant, content_type ) NULLS DISTINCT; CREATE INDEX "Content_text" ON public."Content" USING pgroonga (text); @@ -180,7 +180,7 @@ ADD CONSTRAINT "ResourceAccess_account_uid_fkey" FOREIGN KEY ( CREATE INDEX resource_access_content_local_id_idx ON public."ResourceAccess" (source_local_id, space_id); --- note that I cannot have a foreign key for Content because the variant is part of the unique key. +-- note that I cannot have a foreign key for Content because variant and content_type are part of the unique key. GRANT ALL ON TABLE public."ResourceAccess" TO authenticated; GRANT ALL ON TABLE public."ResourceAccess" TO service_role; @@ -429,6 +429,9 @@ BEGIN IF content.metadata IS NULL then content.metadata := '{}'; END IF; + IF content.content_type IS NULL THEN + content.content_type := 'text/plain'; + END IF; RETURN content; END; $$; @@ -644,7 +647,7 @@ BEGIN db_content.part_of_id, db_content.content_type ) - ON CONFLICT (space_id, source_local_id, variant) DO UPDATE SET + ON CONFLICT (space_id, source_local_id, variant, content_type) DO UPDATE SET document_id = COALESCE(db_content.document_id, EXCLUDED.document_id), author_id = COALESCE(db_content.author_id, EXCLUDED.author_id), creator_id = COALESCE(db_content.creator_id, EXCLUDED.creator_id), @@ -653,8 +656,7 @@ BEGIN metadata = COALESCE(db_content.metadata, EXCLUDED.metadata), scale = COALESCE(db_content.scale, EXCLUDED.scale), last_modified = COALESCE(db_content.last_modified, EXCLUDED.last_modified), - part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id), - content_type = COALESCE(db_content.content_type, EXCLUDED.content_type) + part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id) RETURNING id INTO STRICT upsert_id; IF model(embedding_inline(local_content)) IS NOT NULL THEN PERFORM public.upsert_content_embedding(upsert_id, model(embedding_inline(local_content)), vector(embedding_inline(local_content))); diff --git a/packages/database/supabase/schemas/embedding.sql b/packages/database/supabase/schemas/embedding.sql index 5a5346491..43801cf5d 100644 --- a/packages/database/supabase/schemas/embedding.sql +++ b/packages/database/supabase/schemas/embedding.sql @@ -44,7 +44,8 @@ ct.space_id, ct.last_modified, ct.part_of_id, emb.model, -emb.vector +emb.vector, +ct.content_type FROM public."Content" AS ct JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id) LEFT OUTER JOIN public.my_accessible_resources () AS ra USING (space_id, source_local_id) From 53609fdbe105b9885c85b61ff493131a2f3a9d57 Mon Sep 17 00:00:00 2001 From: Michael Gartner Date: Sun, 5 Jul 2026 23:44:13 -0600 Subject: [PATCH 2/3] Fix content type reader filters --- apps/obsidian/src/utils/importNodes.ts | 62 ++++++++++++++++--- apps/obsidian/src/utils/publishNode.ts | 3 + .../features/step-definitions/stepdefs.ts | 12 ++-- 3 files changed, 64 insertions(+), 13 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 06ba93edd..9fceac3dd 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -31,6 +31,28 @@ type PublishedNode = { authorId: number | undefined; }; +type ObsidianNativeVariant = "direct" | "full"; + +const PLAIN_TEXT_CONTENT_TYPE = "text/plain"; +const OBSIDIAN_MARKDOWN_CONTENT_TYPE = "text/obsidian+markdown"; + +const getObsidianNativeContentType = ( + variant: ObsidianNativeVariant, +): string => + variant === "direct" + ? PLAIN_TEXT_CONTENT_TYPE + : OBSIDIAN_MARKDOWN_CONTENT_TYPE; + +const isObsidianNativeContentRow = ({ + variant, + content_type, +}: { + variant: string | null; + content_type: string | null; +}): boolean => + (variant === "direct" && content_type === PLAIN_TEXT_CONTENT_TYPE) || + (variant === "full" && content_type === OBSIDIAN_MARKDOWN_CONTENT_TYPE); + export const getPublishedNodesForGroups = async ({ client, groupIds, @@ -49,9 +71,14 @@ export const getPublishedNodesForGroups = async ({ const { data, error } = await client .from("my_contents") .select( - "source_local_id, space_id, text, created, last_modified, variant, metadata, author_id", + "source_local_id, space_id, text, created, last_modified, variant, content_type, metadata, author_id", ) - .neq("space_id", currentSpaceId); + .neq("space_id", currentSpaceId) + .in("variant", ["direct", "full"]) + .in("content_type", [ + PLAIN_TEXT_CONTENT_TYPE, + OBSIDIAN_MARKDOWN_CONTENT_TYPE, + ]); if (error) { console.error("Error fetching published nodes:", error); @@ -69,6 +96,7 @@ export const getPublishedNodesForGroups = async ({ created: string | null; last_modified: string | null; variant: string | null; + content_type: string | null; author_id: number | null; metadata: Json; }; @@ -76,6 +104,7 @@ export const getPublishedNodesForGroups = async ({ const key = (r: Row) => `${r.space_id ?? ""}\t${r.source_local_id ?? ""}`; const groups = new Map(); for (const row of data as Row[]) { + if (!isObsidianNativeContentRow(row)) continue; if (row.source_local_id == null || row.space_id == null) continue; const k = key(row); if (!groups.has(k)) groups.set(k, []); @@ -92,7 +121,10 @@ export const getPublishedNodesForGroups = async ({ const latest = withDate.reduce((a, b) => (a.last_modified ?? "") >= (b.last_modified ?? "") ? a : b, ); - const direct = rows.find((r) => r.variant === "direct"); + const direct = rows.find( + (r) => + r.variant === "direct" && r.content_type === PLAIN_TEXT_CONTENT_TYPE, + ); const text = direct?.text ?? latest.text ?? ""; const createdAt = latest.created ? new Date(latest.created + "Z").valueOf() @@ -250,6 +282,7 @@ export const fetchNodeContent = async ({ .eq("source_local_id", nodeInstanceId) .eq("space_id", spaceId) .eq("variant", variant) + .eq("content_type", getObsidianNativeContentType(variant)) .maybeSingle(); if (error || !data || data.text == null) { @@ -284,6 +317,7 @@ export const fetchNodeContentWithMetadata = async ({ .eq("source_local_id", nodeInstanceId) .eq("space_id", spaceId) .eq("variant", variant) + .eq("content_type", getObsidianNativeContentType(variant)) .maybeSingle(); if (error || !data || data.text == null) { @@ -325,10 +359,16 @@ const fetchNodeContentForImport = async ({ } | null> => { const { data, error } = await client .from("my_contents") - .select("text, created, last_modified, variant, metadata, author_id") + .select( + "text, created, last_modified, variant, content_type, metadata, author_id", + ) .eq("source_local_id", nodeInstanceId) .eq("space_id", spaceId) - .in("variant", ["direct", "full"]); + .in("variant", ["direct", "full"]) + .in("content_type", [ + PLAIN_TEXT_CONTENT_TYPE, + OBSIDIAN_MARKDOWN_CONTENT_TYPE, + ]); if (error) { console.error("Error fetching node content for import:", error); @@ -341,10 +381,16 @@ const fetchNodeContentForImport = async ({ last_modified: string | null; author_id: number | null; variant: string | null; + content_type: string | null; metadata: Json; }>; - const direct = rows.find((r) => r.variant === "direct"); - const full = rows.find((r) => r.variant === "full"); + const direct = rows.find( + (r) => r.variant === "direct" && r.content_type === PLAIN_TEXT_CONTENT_TYPE, + ); + const full = rows.find( + (r) => + r.variant === "full" && r.content_type === OBSIDIAN_MARKDOWN_CONTENT_TYPE, + ); const authorId = full?.author_id ?? direct?.author_id ?? null; if ( @@ -395,6 +441,7 @@ export const getSourceContentDates = async ({ .eq("source_local_id", nodeInstanceId) .eq("space_id", spaceId) .eq("variant", "direct") + .eq("content_type", PLAIN_TEXT_CONTENT_TYPE) .maybeSingle(); if (error || !data) return null; return { @@ -1525,6 +1572,7 @@ export const refreshImportedFile = async ({ .eq("space_id", spaceId) .eq("source_local_id", frontmatter.nodeInstanceId) .eq("variant", "direct") + .eq("content_type", PLAIN_TEXT_CONTENT_TYPE) .maybeSingle(); const metadata = metadataResp.data?.metadata; const filePath: string | undefined = diff --git a/apps/obsidian/src/utils/publishNode.ts b/apps/obsidian/src/utils/publishNode.ts index e18cfecf4..0a3c5d9d3 100644 --- a/apps/obsidian/src/utils/publishNode.ts +++ b/apps/obsidian/src/utils/publishNode.ts @@ -23,6 +23,8 @@ import type { DiscourseNodeInVault } from "./getDiscourseNodes"; import type { SupabaseContext } from "./supabaseContext"; import type { TablesInsert } from "@repo/database/dbTypes"; +const OBSIDIAN_MARKDOWN_CONTENT_TYPE = "text/obsidian+markdown"; + export const getPublishedToGroups = ( frontmatter: FrontMatterCache | Record, ): string[] => { @@ -428,6 +430,7 @@ export const publishNodeToGroup = async ({ .eq("source_local_id", nodeId) .eq("space_id", spaceId) .eq("variant", "full") + .eq("content_type", OBSIDIAN_MARKDOWN_CONTENT_TYPE) .maybeSingle(); if (idResponse.error || !idResponse.data) { throw idResponse.error || new Error("no data while fetching node"); diff --git a/packages/database/features/step-definitions/stepdefs.ts b/packages/database/features/step-definitions/stepdefs.ts index 36aad239a..4f0f087e8 100644 --- a/packages/database/features/step-definitions/stepdefs.ts +++ b/packages/database/features/step-definitions/stepdefs.ts @@ -308,15 +308,14 @@ Then( }, ); +/* eslint-disable max-params -- Cucumber inspects function.length for step arity. */ Then( "a user logged in space {word} should see {int} content rows with variant {string} and content type {string}", async ( - ...[spaceName, expectedCount, variant, contentType]: [ - string, - number, - ContentVariant, - string, - ] + spaceName: string, + expectedCount: number, + variant: ContentVariant, + contentType: string, ) => { const localRefs = (world.localRefs || {}) as LocalRefsType; const spaceId = localRefs[spaceName]; @@ -331,6 +330,7 @@ Then( assert.equal(response.count, expectedCount); }, ); +/* eslint-enable max-params */ // invoke the upsert_accounts_in_space function, expects json Given( From 5b1ca3eb3dc3c0f7c295ae688b54701d28fef3e3 Mon Sep 17 00:00:00 2001 From: Michael Gartner Date: Mon, 6 Jul 2026 00:00:01 -0600 Subject: [PATCH 3/3] Defer Obsidian reader filters --- apps/obsidian/src/utils/importNodes.ts | 62 +++----------------------- apps/obsidian/src/utils/publishNode.ts | 3 -- 2 files changed, 7 insertions(+), 58 deletions(-) diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 9fceac3dd..06ba93edd 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -31,28 +31,6 @@ type PublishedNode = { authorId: number | undefined; }; -type ObsidianNativeVariant = "direct" | "full"; - -const PLAIN_TEXT_CONTENT_TYPE = "text/plain"; -const OBSIDIAN_MARKDOWN_CONTENT_TYPE = "text/obsidian+markdown"; - -const getObsidianNativeContentType = ( - variant: ObsidianNativeVariant, -): string => - variant === "direct" - ? PLAIN_TEXT_CONTENT_TYPE - : OBSIDIAN_MARKDOWN_CONTENT_TYPE; - -const isObsidianNativeContentRow = ({ - variant, - content_type, -}: { - variant: string | null; - content_type: string | null; -}): boolean => - (variant === "direct" && content_type === PLAIN_TEXT_CONTENT_TYPE) || - (variant === "full" && content_type === OBSIDIAN_MARKDOWN_CONTENT_TYPE); - export const getPublishedNodesForGroups = async ({ client, groupIds, @@ -71,14 +49,9 @@ export const getPublishedNodesForGroups = async ({ const { data, error } = await client .from("my_contents") .select( - "source_local_id, space_id, text, created, last_modified, variant, content_type, metadata, author_id", + "source_local_id, space_id, text, created, last_modified, variant, metadata, author_id", ) - .neq("space_id", currentSpaceId) - .in("variant", ["direct", "full"]) - .in("content_type", [ - PLAIN_TEXT_CONTENT_TYPE, - OBSIDIAN_MARKDOWN_CONTENT_TYPE, - ]); + .neq("space_id", currentSpaceId); if (error) { console.error("Error fetching published nodes:", error); @@ -96,7 +69,6 @@ export const getPublishedNodesForGroups = async ({ created: string | null; last_modified: string | null; variant: string | null; - content_type: string | null; author_id: number | null; metadata: Json; }; @@ -104,7 +76,6 @@ export const getPublishedNodesForGroups = async ({ const key = (r: Row) => `${r.space_id ?? ""}\t${r.source_local_id ?? ""}`; const groups = new Map(); for (const row of data as Row[]) { - if (!isObsidianNativeContentRow(row)) continue; if (row.source_local_id == null || row.space_id == null) continue; const k = key(row); if (!groups.has(k)) groups.set(k, []); @@ -121,10 +92,7 @@ export const getPublishedNodesForGroups = async ({ const latest = withDate.reduce((a, b) => (a.last_modified ?? "") >= (b.last_modified ?? "") ? a : b, ); - const direct = rows.find( - (r) => - r.variant === "direct" && r.content_type === PLAIN_TEXT_CONTENT_TYPE, - ); + const direct = rows.find((r) => r.variant === "direct"); const text = direct?.text ?? latest.text ?? ""; const createdAt = latest.created ? new Date(latest.created + "Z").valueOf() @@ -282,7 +250,6 @@ export const fetchNodeContent = async ({ .eq("source_local_id", nodeInstanceId) .eq("space_id", spaceId) .eq("variant", variant) - .eq("content_type", getObsidianNativeContentType(variant)) .maybeSingle(); if (error || !data || data.text == null) { @@ -317,7 +284,6 @@ export const fetchNodeContentWithMetadata = async ({ .eq("source_local_id", nodeInstanceId) .eq("space_id", spaceId) .eq("variant", variant) - .eq("content_type", getObsidianNativeContentType(variant)) .maybeSingle(); if (error || !data || data.text == null) { @@ -359,16 +325,10 @@ const fetchNodeContentForImport = async ({ } | null> => { const { data, error } = await client .from("my_contents") - .select( - "text, created, last_modified, variant, content_type, metadata, author_id", - ) + .select("text, created, last_modified, variant, metadata, author_id") .eq("source_local_id", nodeInstanceId) .eq("space_id", spaceId) - .in("variant", ["direct", "full"]) - .in("content_type", [ - PLAIN_TEXT_CONTENT_TYPE, - OBSIDIAN_MARKDOWN_CONTENT_TYPE, - ]); + .in("variant", ["direct", "full"]); if (error) { console.error("Error fetching node content for import:", error); @@ -381,16 +341,10 @@ const fetchNodeContentForImport = async ({ last_modified: string | null; author_id: number | null; variant: string | null; - content_type: string | null; metadata: Json; }>; - const direct = rows.find( - (r) => r.variant === "direct" && r.content_type === PLAIN_TEXT_CONTENT_TYPE, - ); - const full = rows.find( - (r) => - r.variant === "full" && r.content_type === OBSIDIAN_MARKDOWN_CONTENT_TYPE, - ); + const direct = rows.find((r) => r.variant === "direct"); + const full = rows.find((r) => r.variant === "full"); const authorId = full?.author_id ?? direct?.author_id ?? null; if ( @@ -441,7 +395,6 @@ export const getSourceContentDates = async ({ .eq("source_local_id", nodeInstanceId) .eq("space_id", spaceId) .eq("variant", "direct") - .eq("content_type", PLAIN_TEXT_CONTENT_TYPE) .maybeSingle(); if (error || !data) return null; return { @@ -1572,7 +1525,6 @@ export const refreshImportedFile = async ({ .eq("space_id", spaceId) .eq("source_local_id", frontmatter.nodeInstanceId) .eq("variant", "direct") - .eq("content_type", PLAIN_TEXT_CONTENT_TYPE) .maybeSingle(); const metadata = metadataResp.data?.metadata; const filePath: string | undefined = diff --git a/apps/obsidian/src/utils/publishNode.ts b/apps/obsidian/src/utils/publishNode.ts index 0a3c5d9d3..e18cfecf4 100644 --- a/apps/obsidian/src/utils/publishNode.ts +++ b/apps/obsidian/src/utils/publishNode.ts @@ -23,8 +23,6 @@ import type { DiscourseNodeInVault } from "./getDiscourseNodes"; import type { SupabaseContext } from "./supabaseContext"; import type { TablesInsert } from "@repo/database/dbTypes"; -const OBSIDIAN_MARKDOWN_CONTENT_TYPE = "text/obsidian+markdown"; - export const getPublishedToGroups = ( frontmatter: FrontMatterCache | Record, ): string[] => { @@ -430,7 +428,6 @@ export const publishNodeToGroup = async ({ .eq("source_local_id", nodeId) .eq("space_id", spaceId) .eq("variant", "full") - .eq("content_type", OBSIDIAN_MARKDOWN_CONTENT_TYPE) .maybeSingle(); if (idResponse.error || !idResponse.data) { throw idResponse.error || new Error("no data while fetching node");