From 66670b58a9c83cae3be6836f6b4b7c74f94bada1 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 01:17:48 +0900 Subject: [PATCH 1/2] Fix Markdown heading link resolution (#4846) --- DEVELOPER_GUIDE.md | 6 +- USER_GUIDE.md | 8 +- changelog.d/unreleased/4846.fixed.md | 28 +++ .../Database/DbSymbolReader.Definitions.cs | 22 ++- .../Database/DbSymbolReader.Search.cs | 48 +++-- .../Database/DbWriter.FoldBackfill.cs | 98 +++++++++- .../DbWriter.ReferenceGraphRefreshScope.cs | 2 +- src/CodeIndex/Database/DbWriter.References.cs | 35 +++- .../Indexer/MarkdownAnchorIdentity.cs | 65 +++++++ .../MarkupSchemaReferenceExtractor.cs | 174 +++++++++++------- .../ReferenceExtractor.CoreReferenceLoop.cs | 6 +- .../Symbols/SymbolExtractor.Contracts.cs | 4 +- .../Indexer/Symbols/SymbolExtractor.Markup.cs | 146 +++++---------- src/CodeIndex/Models/SymbolKindCatalog.cs | 1 + tests/CodeIndex.Tests/DatabaseTests.cs | 4 +- .../DbReaderFileDependencyTests.cs | 119 ++++++++++++ .../ReferenceExtractorTests.cs | 10 +- tests/CodeIndex.Tests/SymbolExtractorTests.cs | 69 +++++-- 18 files changed, 612 insertions(+), 233 deletions(-) create mode 100644 changelog.d/unreleased/4846.fixed.md create mode 100644 src/CodeIndex/Indexer/MarkdownAnchorIdentity.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 561b4efbf..ec7805fca 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -638,6 +638,7 @@ Do not add mutable static caches, shared `StringBuilder` instances, reused `Matc |---|---|---| | `accessor` | Accessor declarations when extracted separately from their owning property | Search/filter symbol | | `add` | Dockerfile `ADD` destination paths | Dependency/file-flow search symbol | +| `anchor` | Explicit Markdown HTML anchors | Definition target for path-scoped Markdown fragment references | | `annotation` | Annotation declarations or annotation-like language constructs | Metadata/search symbol | | `async_function` | JavaScript/TypeScript async function declarations | Callable definition; participates in callers/callees through reference rows | | `async_generator` | JavaScript/TypeScript async generator declarations | Callable definition; participates in callers/callees through reference rows | @@ -659,7 +660,7 @@ Do not add mutable static caches, shared `StringBuilder` instances, reused `Matc | `file_module` | File-scoped module/package declarations | Namespace-like context symbol | | `function` | Functions, methods, constructors, delegates, tasks, and callable bindings that do not have a narrower kind | Primary callable definition; participates in callers/callees through reference rows | | `generator` | JavaScript/TypeScript generator declarations | Callable definition; participates in callers/callees through reference rows | -| `heading` | Markdown headings and language section markers such as C# regions, Python module docstrings, and JavaScript/TypeScript `@module` docblocks | Outline symbol | +| `heading` | Markdown headings and language section markers such as C# regions, Python module docstrings, and JavaScript/TypeScript `@module` docblocks | Outline symbol; Markdown headings are definition targets for path-scoped fragment references | | `hook` | JavaScript/TypeScript React custom hook bindings | Callable-like search/filter symbol | | `implements` | Razor `@implements` directives | Context/search symbol | | `import` | Imports, using directives, aliases, and package includes | Search/filter symbol | @@ -3784,6 +3785,7 @@ filter、downstream JSON consumer が同じ値を理解できるようにして |---|---|---| | `accessor` | owning property から別 symbol として抽出される accessor declaration | Search/filter symbol | | `add` | Dockerfile `ADD` の destination path | dependency / file-flow search symbol | +| `anchor` | Markdown 内の明示的な HTML anchor | path に限定した Markdown fragment reference の定義対象 | | `annotation` | annotation declaration または annotation-like な言語構文 | Metadata/search symbol | | `async_function` | JavaScript / TypeScript の async function declaration | Callable definition。reference row 経由で callers/callees に参加 | | `async_generator` | JavaScript / TypeScript の async generator declaration | Callable definition。reference row 経由で callers/callees に参加 | @@ -3805,7 +3807,7 @@ filter、downstream JSON consumer が同じ値を理解できるようにして | `file_module` | file-scoped module / package declaration | Namespace-like context symbol | | `function` | 関数、method、constructor、delegate、task、およびより狭い kind がない callable binding | Primary callable definition。reference row 経由で callers/callees に参加 | | `generator` | JavaScript / TypeScript generator declaration | Callable definition。reference row 経由で callers/callees に参加 | -| `heading` | Markdown heading、C# region、Python module docstring、JavaScript / TypeScript `@module` docblock などの language section marker | Outline symbol | +| `heading` | Markdown heading、C# region、Python module docstring、JavaScript / TypeScript `@module` docblock などの language section marker | Outline symbol。Markdown heading は path に限定した fragment reference の定義対象 | | `hook` | JavaScript / TypeScript React custom hook binding | Callable-like search/filter symbol | | `implements` | Razor `@implements` directive | Context/search symbol | | `import` | import、using directive、alias、package include | Search/filter symbol | diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 65e538490..a109a4e9e 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2478,7 +2478,7 @@ All indexed languages are searchable through FTS5. Rows with **Symbols = yes** a - Dynamic/declarative graph languages: Crystal, Groovy, Tcl, and Prolog expose conservative declarations, imports, and call relationships. Crystal, Groovy, and Prolog parenthesized calls use the shared extractor; command-style calls are limited to callables declared in the same file. Tcl recognizes command substitutions and common control-command script arguments without treating ordinary `name()` words as calls, while Tcl proc / Prolog predicate bodies preserve caller containers. An index created before this graph contract reports `reference_graph_complete=false` and `graph_data_current=false` with `dynamic_reference_graph_contract_stale`; rerun `cdidx index ` to refresh affected rows before treating absent edges as authoritative. - Scientific and native-extension graphs: Julia, MATLAB, Nim, D, Cython, and Ada emit bounded language-aware import/module, base/type, and call references. Julia macro invocations and Ada procedure-style calls without parentheses are also represented. -- Markdown, JSON/YAML, and CSS: Markdown heading and local-anchor symbols are indexed; JSON/YAML configuration keys are indexed as structural key paths; CSS variables, placeholders, and `@extend` references are indexed. +- Markdown, JSON/YAML, and CSS: Markdown headings and explicit HTML anchors are indexed as definitions, while local and cross-document fragment links are indexed as path-scoped references; JSON/YAML configuration keys are indexed as structural key paths; CSS variables, placeholders, and `@extend` references are indexed. - Dockerfile, Assembly, Common Lisp, and Racket: `ARG` build args, labels/PROC/MACRO blocks, package/module forms, definitions, classes/structs, requires, and provides are surfaced as symbols where applicable. - Shell, PowerShell, and Batch: command-style function calls, functions/filters, classes/enums, imports, labels, `goto` / `call` targets, and inline control-flow forms are indexed where the language supports them. - C# and Java: modern C# partial members remain visible to `symbols`, `definition`, and `outline`; Java sealed `permits` lists are recorded as `type_reference` graph edges. @@ -2526,7 +2526,7 @@ entries with the unsupported capability, an explanatory message, and | Shell / PowerShell / Batch / Makefile / CMake / Justfile / MSBuild / Gradle | functions, labels, targets, recipes, tasks, imports where applicable | command-style calls, target dependencies, and control-flow targets | Runtime command construction is not resolved. | | Solution / application manifest | solution projects and manifest identity/settings | solution project references; application manifests are symbol-only | `.sln` project paths are graph edges for repository structure; use `symbols --lang app_manifest` for Windows manifest metadata. | | SQL / Terraform / Dockerfile | statements/resources/stages/labels | table/resource/stage references, Dockerfile stage dependencies, Terraform dotted refs | SQL hotspot grouping defaults to statements; Dockerfile `COPY --from=` follows named stages. | -| Markdown / HTML / CSS / Sass / Stylus / XML / XAML / GraphQL / Protobuf | headings, anchors, selectors, UI elements, generic XML element/attribute paths, schema types/messages where supported | links/assets/components, local anchors, CSS/Sass/Stylus imports, variables, mixins/functions, XAML resources/bindings/handlers, schema references where supported | Generic non-XAML XML emits bounded structural symbols; use `search` for prose and generated markup. | +| Markdown / HTML / CSS / Sass / Stylus / XML / XAML / GraphQL / Protobuf | headings, explicit anchors, selectors, UI elements, generic XML element/attribute paths, schema types/messages where supported | links/assets/components, path-scoped local and cross-document fragments, CSS/Sass/Stylus imports, variables, mixins/functions, XAML resources/bindings/handlers, schema references where supported | Markdown fragment references resolve only against headings or explicit anchors in the linked document. Generic non-XAML XML emits bounded structural symbols; use `search` for prose and generated markup. | | Dependency manifests / lockfiles | none | none | Use `--lang dependency_manifest` or `--lang dependency_lock` for dependency/security audits. | | Other indexed text formats | file/chunk search only unless `languages` reports symbols | no graph unless `languages` reports support | `cdidx search "literal" --lang yaml` is the reliable fallback. | @@ -5568,7 +5568,7 @@ indexing はファイル単位の SQLite transaction を commit します。長 - 動的・宣言型言語の graph 対応: Crystal、Groovy、Tcl、Prolog は保守的な宣言、import、call relationship を公開します。Crystal、Groovy、Prolog の括弧付き call は共通 extractor を使い、command-style call は同一ファイルで宣言済みの callable に限定します。Tcl は通常の `name()` word を call とみなさず、command substitution と主要な制御 command の script 引数を認識し、Tcl proc / Prolog predicate の本体では caller container を保持します。 この graph contract より前に作成された index は `dynamic_reference_graph_contract_stale` とともに `reference_graph_complete=false`、`graph_data_current=false` を報告します。欠落 edge を authoritative とみなす前に `cdidx index ` を再実行して対象 row を更新してください。 - 科学技術・ネイティブ拡張言語のグラフ: Julia、MATLAB、Nim、D、Cython、Ada は、言語構文に応じた import/module、基底型/type、call 参照を上限付きで出力します。Julia の macro invocation と、括弧を伴わない Ada の procedure call も記録します。 -- Markdown、JSON/YAML、CSS: Markdown の heading / local anchor、JSON/YAML の configuration key path、CSS の variable、placeholder、`@extend` をシンボルとして扱います。 +- Markdown、JSON/YAML、CSS: Markdown の heading と明示的な HTML anchor は定義として、同一文書・文書間の fragment link は対象 path に限定した参照として索引します。JSON/YAML の configuration key path、CSS の variable、placeholder、`@extend` もシンボルとして扱います。 - Dockerfile、Assembly、Common Lisp、Racket: `ARG` build arg、label、PROC/MACRO、package/module form、definition、class/struct、require/provide を必要に応じて表面化します。 - Shell、PowerShell、Batch: command-style function call、function/filter、class/enum、import、label、`goto` / `call` target、inline control-flow を言語仕様に合わせて索引します。 - C# と Java: C# の近年の partial member は `symbols`、`definition`、`outline` から見えます。Java の sealed `permits` list は `type_reference` graph edge として記録します。 @@ -5610,7 +5610,7 @@ indexing はファイル単位の SQLite transaction を commit します。長 | Shell / PowerShell / Batch / Makefile / CMake / Justfile / MSBuild / Gradle | function、label、target、recipe、task、対応言語の import | command-style call、target dependency、control-flow target | runtime で組み立てられる command は解決しません。 | | ソリューション / アプリケーションマニフェスト | solution project、manifest identity / setting | `.sln` の project reference。manifest は symbol-only | `.sln` の project path はリポジトリ構造の graph edge です。Windows manifest metadata は `symbols --lang app_manifest` で確認できます。 | | SQL / Terraform / Dockerfile | statement/resource/stage/label | table/resource/stage reference、Dockerfile stage dependency、Terraform dotted refs | SQL hotspot grouping は既定で statement、Dockerfile `COPY --from=` は named stage を追跡します。 | -| Markdown / HTML / CSS / Sass / Stylus / XML / XAML / GraphQL / Protobuf | heading、anchor、selector、UI element、汎用 XML の要素・属性パス、対応 schema type/message | link/asset/component、local anchor、CSS/Sass/Stylus の import・variable・mixin/function、XAML resource / binding / handler、対応 schema reference | 汎用の非 XAML XML は上限付きの構造シンボルを出力します。prose や generated markup には `search` を使ってください。 | +| Markdown / HTML / CSS / Sass / Stylus / XML / XAML / GraphQL / Protobuf | heading、明示的な anchor、selector、UI element、汎用 XML の要素・属性パス、対応 schema type/message | link/asset/component、path に限定した同一文書・文書間の fragment、CSS/Sass/Stylus の import・variable・mixin/function、XAML resource / binding / handler、対応 schema reference | Markdown の fragment reference は、リンク先文書内の heading または明示的な anchor にのみ解決します。汎用の非 XAML XML は上限付きの構造シンボルを出力します。prose や generated markup には `search` を使ってください。 | | Dependency manifest / lockfile | なし | なし | dependency / security audit には `--lang dependency_manifest` または `--lang dependency_lock` を使います。 | | その他の indexed text format | `languages` が symbol 対応を示す場合を除き file/chunk search のみ | `languages` が graph 対応を示す場合を除きなし | `cdidx search "literal" --lang yaml` が信頼できる fallback です。 | diff --git a/changelog.d/unreleased/4846.fixed.md b/changelog.d/unreleased/4846.fixed.md new file mode 100644 index 000000000..f375fcc39 --- /dev/null +++ b/changelog.d/unreleased/4846.fixed.md @@ -0,0 +1,28 @@ +--- +category: fixed +issues: + - 4846 +affected: + - src/CodeIndex/Indexer/MarkdownAnchorIdentity.cs + - src/CodeIndex/Models/SymbolKindCatalog.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Markup.cs + - src/CodeIndex/Indexer/References/Languages/MarkupSchemaReferenceExtractor.cs + - src/CodeIndex/Database/DbSymbolReader.Definitions.cs + - src/CodeIndex/Database/DbSymbolReader.Search.cs + - src/CodeIndex/Database/DbWriter.FoldBackfill.cs + - src/CodeIndex/Database/DbWriter.References.cs + - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs + - tests/CodeIndex.Tests/DbReaderFileDependencyTests.cs + - tests/CodeIndex.Tests/DatabaseTests.cs + - tests/CodeIndex.Tests/LegacySchemaMigrationTests.cs + - USER_GUIDE.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Markdown fragment links now resolve to path-scoped heading and explicit-anchor definitions (#4846)** — local and cross-document links use normalized fragment identities, duplicate heading slugs remain distinct, and equal heading names in unrelated documents no longer create false dependency edges. This completes the Markdown definition follow-up to #4113. + +## 日本語 + +- **Markdown の fragment link が path に限定された heading と明示的な anchor の定義へ解決されるようになりました (#4846)** — 同一文書・文書間の link は正規化した fragment identity を使い、重複する heading slug を区別し、無関係な文書の同名 heading による誤った依存 edge を作らなくなりました。これは #4113 に続く Markdown definition の修正です。 diff --git a/src/CodeIndex/Database/DbSymbolReader.Definitions.cs b/src/CodeIndex/Database/DbSymbolReader.Definitions.cs index 26ba01a4f..50d2eb562 100644 --- a/src/CodeIndex/Database/DbSymbolReader.Definitions.cs +++ b/src/CodeIndex/Database/DbSymbolReader.Definitions.cs @@ -328,6 +328,12 @@ FROM symbols s var qualifiedSymbolClause = SqlNameResolver.HasQualifier(normalizedQuery) ? BuildQualifiedSymbolMatchSql("query", _foldReady) : null; + var markdownAnchorExactClause = _symbolColumns.Contains("name_folded") + ? "(f.lang = 'markdown' AND s.kind IN ('heading', 'anchor') AND s.name_folded = @queryMarkdownAnchor)" + : "0"; + var markdownAnchorLikeClause = _symbolColumns.Contains("name_folded") + ? " OR (f.lang = 'markdown' AND s.kind IN ('heading', 'anchor') AND s.name_folded LIKE @queryMarkdownAnchorLike ESCAPE '\\')" + : string.Empty; sql += exact ? rustQualifiedParts.QualifiedPath != null ? _foldReady @@ -335,12 +341,12 @@ FROM symbols s : " AND ((s.container_qualified_name = @queryRustContainer COLLATE NOCASE OR s.container_name = @queryRustContainer COLLATE NOCASE) AND s.name = @queryRustLeaf COLLATE NOCASE)" : _foldReady ? allowLeafFallback - ? $" AND ({BuildPersistedFoldedNameMatchSql("s.name_folded", "@query")} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded) OR sql_leaf_name_folded(s.name) = @queryLeafFolded)))" - : $" AND ({BuildPersistedFoldedNameMatchSql("s.name_folded", "@query")} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" + ? $" AND ({BuildPersistedFoldedNameMatchSql("s.name_folded", "@query")} OR {markdownAnchorExactClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded) OR sql_leaf_name_folded(s.name) = @queryLeafFolded)))" + : $" AND ({BuildPersistedFoldedNameMatchSql("s.name_folded", "@query")} OR {markdownAnchorExactClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" : allowLeafFallback - ? " AND (s.name = @query COLLATE NOCASE OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name(s.name) = @queryNormalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @queryLeaf COLLATE NOCASE)))" - : $" AND (s.name = @query COLLATE NOCASE OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name(s.name) = @queryNormalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" - : $" AND (s.name LIKE @query ESCAPE '\\' OR (f.lang = 'sql' AND sql_normalize_name(s.name) LIKE @queryNormalizedLike ESCAPE '\\'){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; + ? $" AND (s.name = @query COLLATE NOCASE OR {markdownAnchorExactClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name(s.name) = @queryNormalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @queryLeaf COLLATE NOCASE)))" + : $" AND (s.name = @query COLLATE NOCASE OR {markdownAnchorExactClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name(s.name) = @queryNormalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" + : $" AND (s.name LIKE @query ESCAPE '\\'{markdownAnchorLikeClause} OR (f.lang = 'sql' AND sql_normalize_name(s.name) LIKE @queryNormalizedLike ESCAPE '\\'){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; } if (kind != null) sql += " AND s.kind = @kind"; @@ -379,6 +385,12 @@ FROM chunks c SqliteCommandPolicy.Add(cmd, "@queryLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(normalizedQuery)) ?? SqlNameResolver.GetLeafName(normalizedQuery)); SqliteCommandPolicy.Add(cmd, "@querySegmentCount", SqlNameResolver.GetSegmentCount(normalizedQuery)); SqliteCommandPolicy.Add(cmd, "@queryNormalizedLike", $"%{EscapeLikeQuery(SqlNameResolver.NormalizeQualifiedName(normalizedQuery))}%"); + if (_symbolColumns.Contains("name_folded")) + { + var markdownAnchorIdentity = MarkdownAnchorIdentity.Normalize(normalizedQuery); + SqliteCommandPolicy.Add(cmd, "@queryMarkdownAnchor", markdownAnchorIdentity); + SqliteCommandPolicy.Add(cmd, "@queryMarkdownAnchorLike", $"%{EscapeLikeQuery(markdownAnchorIdentity)}%"); + } if (SqlNameResolver.HasQualifier(normalizedQuery)) AddQualifiedSymbolQueryParameters(cmd, "query", normalizedQuery); if (rustQualifiedParts.QualifiedPath != null) diff --git a/src/CodeIndex/Database/DbSymbolReader.Search.cs b/src/CodeIndex/Database/DbSymbolReader.Search.cs index 707420db4..f6f9fc111 100644 --- a/src/CodeIndex/Database/DbSymbolReader.Search.cs +++ b/src/CodeIndex/Database/DbSymbolReader.Search.cs @@ -193,8 +193,10 @@ private static void AddQualifiedGraphQueryParameters(SqliteCommand cmd, string q public int CountSearchSymbols(IReadOnlyList? queries, int limit = 20, string? kind = null, string? lang = null, IReadOnlyList? pathPatterns = null, IReadOnlyList? excludePathPatterns = null, bool excludeTests = false, DateTime? since = null, bool exact = false, IReadOnlyList? visibilityFilters = null, IReadOnlyList? excludeVisibilityFilters = null) { + lang = DbReader.NormalizeQueryLanguage(lang); var validQueries = NormalizeSymbolSearchQueries(queries, lang, exact); - if (validQueries != null && validQueries.Count > 1) + if (validQueries != null + && (validQueries.Count > 1 || string.Equals(lang, "markdown", StringComparison.Ordinal))) return SearchSymbols(validQueries, limit, kind, lang, pathPatterns, excludePathPatterns, excludeTests, since, exact, visibilityFilters, excludeVisibilityFilters).Count; using var cmd = _conn.CreateCommand(); @@ -318,6 +320,9 @@ FROM symbols s var rustQualifiedExact = ShouldPreserveRustQualifiedExactQuery(queryValue, lang, exact); var rustQualifiedParts = rustQualifiedExact ? NormalizeRustQualifiedExactQueryParts(queryValue) : default; var allowLeafFallback = !SqlNameResolver.HasQualifier(queryValue); + var markdownAnchorClause = _symbolColumns.Contains("name_folded") + ? $"(f.lang = 'markdown' AND s.kind IN ('heading', 'anchor') AND s.name_folded = @query{idx}MarkdownAnchor)" + : "0"; var qualifiedSymbolClause = SqlNameResolver.HasQualifier(queryValue) ? BuildQualifiedSymbolMatchSql($"query{idx}", _foldReady) : null; @@ -333,18 +338,21 @@ FROM symbols s : $"((s.container_qualified_name = @query{idx}RustContainer COLLATE NOCASE OR s.container_name = @query{idx}RustContainer COLLATE NOCASE) AND s.name = @query{idx}RustLeaf COLLATE NOCASE)"; return _foldReady ? allowLeafFallback - ? $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query{idx}LeafFolded)))" - : $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" + ? $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query{idx}LeafFolded)))" + : $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" : allowLeafFallback - ? $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @query{idx}Leaf COLLATE NOCASE)))" - : $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; + ? $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @query{idx}Leaf COLLATE NOCASE)))" + : $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; })) : string.Join(" OR ", effectiveQueries.Select((queryValue, idx) => { var qualifiedSymbolClause = SqlNameResolver.HasQualifier(queryValue) ? BuildQualifiedSymbolMatchSql($"query{idx}", _foldReady) : null; - return $"(s.name LIKE @query{idx} ESCAPE '\\' OR (f.lang = 'sql' AND sql_normalize_name(s.name) LIKE @query{idx}NormalizedLike ESCAPE '\\'){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; + var markdownAnchorLikeClause = _symbolColumns.Contains("name_folded") + ? $" OR (f.lang = 'markdown' AND s.kind IN ('heading', 'anchor') AND s.name_folded LIKE @query{idx}MarkdownAnchorLike ESCAPE '\\')" + : string.Empty; + return $"(s.name LIKE @query{idx} ESCAPE '\\'{markdownAnchorLikeClause} OR (f.lang = 'sql' AND sql_normalize_name(s.name) LIKE @query{idx}NormalizedLike ESCAPE '\\'){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; })); sql += $" AND ({orClauses})"; } @@ -378,6 +386,12 @@ FROM symbols s SqliteCommandPolicy.Add(cmd, $"@query{i}LeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(value)) ?? SqlNameResolver.GetLeafName(value)); SqliteCommandPolicy.Add(cmd, $"@query{i}SegmentCount", SqlNameResolver.GetSegmentCount(value)); SqliteCommandPolicy.Add(cmd, $"@query{i}NormalizedLike", $"%{EscapeLikeQuery(SqlNameResolver.NormalizeQualifiedName(value))}%"); + if (_symbolColumns.Contains("name_folded")) + { + var markdownAnchorIdentity = MarkdownAnchorIdentity.Normalize(value); + SqliteCommandPolicy.Add(cmd, $"@query{i}MarkdownAnchor", markdownAnchorIdentity); + SqliteCommandPolicy.Add(cmd, $"@query{i}MarkdownAnchorLike", $"%{EscapeLikeQuery(markdownAnchorIdentity)}%"); + } if (SqlNameResolver.HasQualifier(value)) AddQualifiedSymbolQueryParameters(cmd, $"query{i}", value); var swiftBacktickAlias = ComputeSwiftBacktickAlias(value, lang); @@ -599,6 +613,9 @@ FROM symbols s var rustQualifiedExact = ShouldPreserveRustQualifiedExactQuery(queryValue, lang, exact); var rustQualifiedParts = rustQualifiedExact ? NormalizeRustQualifiedExactQueryParts(queryValue) : default; var allowLeafFallback = !SqlNameResolver.HasQualifier(queryValue); + var markdownAnchorClause = _symbolColumns.Contains("name_folded") + ? $"(f.lang = 'markdown' AND s.kind IN ('heading', 'anchor') AND s.name_folded = @query{idx}MarkdownAnchor)" + : "0"; var qualifiedSymbolClause = SqlNameResolver.HasQualifier(queryValue) ? BuildQualifiedSymbolMatchSql($"query{idx}", _foldReady) : null; @@ -614,18 +631,21 @@ FROM symbols s : $"((s.container_qualified_name = @query{idx}RustContainer COLLATE NOCASE OR s.container_name = @query{idx}RustContainer COLLATE NOCASE) AND s.name = @query{idx}RustLeaf COLLATE NOCASE)"; return _foldReady ? allowLeafFallback - ? $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query{idx}LeafFolded)))" - : $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" + ? $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query{idx}LeafFolded)))" + : $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" : allowLeafFallback - ? $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @query{idx}Leaf COLLATE NOCASE)))" - : $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; + ? $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @query{idx}Leaf COLLATE NOCASE)))" + : $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; })) : string.Join(" OR ", effectiveQueries.Select((queryValue, idx) => { var qualifiedSymbolClause = SqlNameResolver.HasQualifier(queryValue) ? BuildQualifiedSymbolMatchSql($"query{idx}", _foldReady) : null; - return $"(s.name LIKE @query{idx} ESCAPE '\\' OR (f.lang = 'sql' AND sql_normalize_name(s.name) LIKE @query{idx}NormalizedLike ESCAPE '\\'){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; + var markdownAnchorLikeClause = _symbolColumns.Contains("name_folded") + ? $" OR (f.lang = 'markdown' AND s.kind IN ('heading', 'anchor') AND s.name_folded LIKE @query{idx}MarkdownAnchorLike ESCAPE '\\')" + : string.Empty; + return $"(s.name LIKE @query{idx} ESCAPE '\\'{markdownAnchorLikeClause} OR (f.lang = 'sql' AND sql_normalize_name(s.name) LIKE @query{idx}NormalizedLike ESCAPE '\\'){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; })); sql += $" AND ({orClauses})"; } @@ -679,6 +699,12 @@ FROM symbols s SqliteCommandPolicy.Add(cmd, $"@query{idx}LeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(effectiveQueries[idx])) ?? SqlNameResolver.GetLeafName(effectiveQueries[idx])); SqliteCommandPolicy.Add(cmd, $"@query{idx}SegmentCount", SqlNameResolver.GetSegmentCount(effectiveQueries[idx])); SqliteCommandPolicy.Add(cmd, $"@query{idx}NormalizedLike", $"%{EscapeLikeQuery(SqlNameResolver.NormalizeQualifiedName(effectiveQueries[idx]))}%"); + if (_symbolColumns.Contains("name_folded")) + { + var markdownAnchorIdentity = MarkdownAnchorIdentity.Normalize(effectiveQueries[idx]); + SqliteCommandPolicy.Add(cmd, $"@query{idx}MarkdownAnchor", markdownAnchorIdentity); + SqliteCommandPolicy.Add(cmd, $"@query{idx}MarkdownAnchorLike", $"%{EscapeLikeQuery(markdownAnchorIdentity)}%"); + } if (SqlNameResolver.HasQualifier(effectiveQueries[idx])) AddQualifiedSymbolQueryParameters(cmd, $"query{idx}", effectiveQueries[idx]); if (rustQualifiedParts.QualifiedPath != null) diff --git a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs index 9f7d00993..ecfa23c2d 100644 --- a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs +++ b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs @@ -94,9 +94,10 @@ private bool AllFoldedColumnsBackfilledCore( public bool AllFoldedColumnValuesMatchCurrentFold() { + var markdownSymbolIdentityFolds = BuildMarkdownSymbolIdentityFoldMap(); var symbols = RentCommand( """ - SELECT s.name, s.name_folded, f.lang + SELECT s.id, s.name, s.name_folded, f.lang, s.kind FROM symbols s JOIN files f ON f.id = s.file_id WHERE s.name IS NOT NULL @@ -107,10 +108,13 @@ WHERE s.name IS NOT NULL using var reader = symbols.ExecuteTrackedReader(); while (reader.TrackedRead()) { - var expected = DbReader.FoldNameForLanguage( - reader.GetString(0), - reader.IsDBNull(2) ? null : reader.GetString(2)); - var actual = reader.IsDBNull(1) ? null : reader.GetString(1); + var expected = FoldPersistedSymbolName( + reader.GetInt64(0), + reader.GetString(1), + reader.IsDBNull(3) ? null : reader.GetString(3), + reader.GetString(4), + markdownSymbolIdentityFolds); + var actual = reader.IsDBNull(2) ? null : reader.GetString(2); if (!string.Equals(actual, expected, StringComparison.Ordinal)) return false; } @@ -372,18 +376,19 @@ private int BackfillSymbolFoldedRows(bool rewriteAll, CancellationToken cancella if (phase == "references") return 0; + var markdownSymbolIdentityFolds = BuildMarkdownSymbolIdentityFoldMap(); var lastSymbolId = rewriteAll ? GetFoldBackfillCheckpoint(FoldBackfillLastSymbolIdMetaKey) : 0; - var rows = new List<(long Id, string Name, string? Lang)>(); + var rows = new List<(long Id, string Name, string? Lang, string Kind)>(); var selectSql = rewriteAll ? """ - SELECT s.id, s.name, f.lang + SELECT s.id, s.name, f.lang, s.kind FROM symbols s JOIN files f ON f.id = s.file_id WHERE s.name IS NOT NULL AND s.id > @lastSymbolId ORDER BY s.id """ : """ - SELECT s.id, s.name, f.lang + SELECT s.id, s.name, f.lang, s.kind FROM symbols s JOIN files f ON f.id = s.file_id WHERE s.name IS NOT NULL AND s.name_folded IS NULL @@ -404,7 +409,8 @@ WHERE s.name IS NOT NULL AND s.name_folded IS NULL rows.Add(( reader.GetInt64(0), reader.GetString(1), - reader.IsDBNull(2) ? null : reader.GetString(2))); + reader.IsDBNull(2) ? null : reader.GetString(2), + reader.GetString(3))); } } finally @@ -429,7 +435,12 @@ WHERE s.name IS NOT NULL AND s.name_folded IS NULL foreach (var row in rows) { cancellationToken.ThrowIfCancellationRequested(); - pFolded.Value = DbReader.FoldNameForLanguage(row.Name, row.Lang); + pFolded.Value = FoldPersistedSymbolName( + row.Id, + row.Name, + row.Lang, + row.Kind, + markdownSymbolIdentityFolds); pId.Value = row.Id; update.ExecuteNonQuery(); if (rewriteAll) @@ -445,6 +456,73 @@ WHERE s.name IS NOT NULL AND s.name_folded IS NULL return rows.Count; } + private Dictionary BuildMarkdownSymbolIdentityFoldMap() + { + var identities = new Dictionary(); + var usedHeadingIdentitiesByFile = new Dictionary>(); + var select = RentCommand( + """ + SELECT s.id, s.file_id, s.kind, s.name + FROM symbols s + JOIN files f ON f.id = s.file_id + WHERE f.lang = 'markdown' + AND s.kind IN ('heading', 'anchor') + AND s.name IS NOT NULL + ORDER BY s.file_id, COALESCE(s.start_line, s.line), s.id + """, + static _ => { }); + try + { + using var reader = select.ExecuteTrackedReader(); + while (reader.TrackedRead()) + { + var symbolId = reader.GetInt64(0); + var fileId = reader.GetInt64(1); + var kind = reader.GetString(2); + var name = reader.GetString(3); + if (kind == "heading") + { + if (!usedHeadingIdentitiesByFile.TryGetValue(fileId, out var usedHeadingIdentities)) + { + usedHeadingIdentities = new HashSet(StringComparer.Ordinal); + usedHeadingIdentitiesByFile.Add(fileId, usedHeadingIdentities); + } + + identities.Add( + symbolId, + MarkdownAnchorIdentity.CreateUniqueHeadingIdentity(name, usedHeadingIdentities)); + } + else + { + identities.Add(symbolId, MarkdownAnchorIdentity.Normalize(name)); + } + } + } + finally + { + ReleaseCommand(select); + } + + return identities; + } + + private static string FoldPersistedSymbolName( + long symbolId, + string name, + string? lang, + string kind, + IReadOnlyDictionary markdownSymbolIdentityFolds) + { + if (lang == "markdown" + && (kind == "heading" || kind == "anchor") + && markdownSymbolIdentityFolds.TryGetValue(symbolId, out var identity)) + { + return identity; + } + + return DbReader.FoldNameForLanguage(name, lang); + } + private int BackfillReferenceFoldedRows(bool rewriteAll, CancellationToken cancellationToken) { var lastReferenceId = rewriteAll ? GetFoldBackfillCheckpoint(FoldBackfillLastReferenceIdMetaKey) : 0; diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index 1f206d70d..088e1f880 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -157,7 +157,7 @@ private static string BuildScopedReferenceCandidatesSql() const string fullInstantiateNamePredicateSql = "AND s.name_folded IS NOT NULL"; const string fullCSharpTypeSymbolSourceSql = "FROM symbols AS type_symbol"; const string fullCSharpTypeNamePredicateSql = "AND type_symbol.name_folded IS NOT NULL"; - const int expectedReferenceSourceCount = 12; + const int expectedReferenceSourceCount = 14; if (CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullDeleteSql) != 1 || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullReferenceSourceSql) diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index f039b9412..470061945 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -412,6 +412,34 @@ HAVING COUNT(DISTINCT target_file.path || char(31) || private static string RefreshReferenceCandidatesSql => $""" DELETE FROM symbol_reference_candidates; + INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) + SELECT r.id, target.id, 0 + FROM symbol_references AS r + JOIN files AS source_file ON source_file.id = r.file_id + JOIN symbols AS target + ON target.file_id = source_file.id + AND target.name_folded = r.symbol_name_folded + AND target.kind IN ('heading', 'anchor') + WHERE source_file.lang = 'markdown' + AND r.reference_kind = 'reference' + AND r.target_qualifier IS NULL; + + INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) + SELECT r.id, target.id, 0 + FROM symbol_references AS r + JOIN files AS source_file ON source_file.id = r.file_id + JOIN files AS target_file + ON target_file.lang = 'markdown' + AND target_file.path = + markdown_resolve_path(source_file.path, r.target_qualifier) + JOIN symbols AS target + ON target.file_id = target_file.id + AND target.name_folded = r.symbol_name_folded + AND target.kind IN ('heading', 'anchor') + WHERE source_file.lang = 'markdown' + AND r.reference_kind = 'reference' + AND r.target_qualifier IS NOT NULL; + INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) SELECT r.id, s.id, 0 FROM symbol_references AS r @@ -444,6 +472,7 @@ THEN r.symbol_name_folded || 'attribute' END OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND {CSharpTypeReferenceCandidatePredicateSql} + AND source_file.lang <> 'markdown' AND r.target_qualifier IS NOT NULL AND r.target_qualifier NOT LIKE char(31) || 'receiver:%' AND ( @@ -532,6 +561,7 @@ THEN r.symbol_name_folded || 'attribute' END OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND {CSharpTypeReferenceCandidatePredicateSql} + AND source_file.lang <> 'markdown' AND r.target_qualifier IS NULL AND s.file_id = r.file_id AND source.container_name IS NOT NULL @@ -563,6 +593,7 @@ THEN r.symbol_name_folded || 'attribute' END OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND {CSharpTypeReferenceCandidatePredicateSql} + AND source_file.lang <> 'markdown' AND r.target_qualifier IS NULL AND source.container_qualified_name IS NOT NULL AND source.container_qualified_name <> '' @@ -589,6 +620,7 @@ THEN r.symbol_name_folded || 'attribute' END OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND {CSharpTypeReferenceCandidatePredicateSql} + AND source_file.lang <> 'markdown' AND r.target_qualifier IS NULL AND s.file_id = r.file_id AND NOT EXISTS ( @@ -614,6 +646,7 @@ THEN r.symbol_name_folded || 'attribute' END OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND {CSharpTypeReferenceCandidatePredicateSql} + AND source_file.lang <> 'markdown' AND r.target_qualifier IS NULL AND source.container_name IS NOT NULL AND source.container_name <> '' @@ -700,7 +733,7 @@ AND target_file.lang IN ('matlab', 'objc') AND target_file.path || char(31) || COALESCE(target.container_qualified_name, target.container_name, '') || char(31) || COALESCE(target.name, '') = unique_family.family_key - WHERE source_file.lang <> 'csharp' + WHERE source_file.lang NOT IN ('csharp', 'markdown') AND ( r.target_qualifier IS NULL OR source_file.lang IN ( diff --git a/src/CodeIndex/Indexer/MarkdownAnchorIdentity.cs b/src/CodeIndex/Indexer/MarkdownAnchorIdentity.cs new file mode 100644 index 000000000..12d95bc4b --- /dev/null +++ b/src/CodeIndex/Indexer/MarkdownAnchorIdentity.cs @@ -0,0 +1,65 @@ +using System.Text; + +namespace CodeIndex.Indexer; + +internal static class MarkdownAnchorIdentity +{ + internal static string Normalize(string value) + { + var anchor = value.Trim(); + if (anchor.Length >= 2 && anchor[0] == '<' && anchor[^1] == '>') + anchor = anchor[1..^1].Trim(); + anchor = anchor.TrimStart('#').Trim(); + if (anchor.Length == 0) + return string.Empty; + + try + { + anchor = Uri.UnescapeDataString(anchor); + } + catch (UriFormatException) + { + // Preserve malformed fragments as unresolved evidence instead of dropping them. + // 不正な fragment も削除せず、未解決の参照根拠として保持する。 + } + + anchor = anchor.Normalize(NormalizationForm.FormKC); + var builder = new StringBuilder(anchor.Length); + var previousDash = false; + foreach (var originalRune in anchor.EnumerateRunes()) + { + var rune = Rune.ToLowerInvariant(originalRune); + if (Rune.IsLetterOrDigit(rune) || rune.Value is '_' or '-') + { + builder.Append(rune.ToString()); + previousDash = rune.Value == '-'; + } + else if (Rune.IsWhiteSpace(rune) && builder.Length > 0 && !previousDash) + { + builder.Append('-'); + previousDash = true; + } + } + + while (builder.Length > 0 && builder[^1] == '-') + builder.Length--; + + return builder.ToString(); + } + + internal static string CreateUniqueHeadingIdentity(string headingText, HashSet usedIdentities) + { + var baseIdentity = Normalize(headingText); + if (baseIdentity.Length == 0) + return string.Empty; + if (usedIdentities.Add(baseIdentity)) + return baseIdentity; + + for (var suffix = 1; ; suffix++) + { + var candidate = $"{baseIdentity}-{suffix}"; + if (usedIdentities.Add(candidate)) + return candidate; + } + } +} diff --git a/src/CodeIndex/Indexer/References/Languages/MarkupSchemaReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/MarkupSchemaReferenceExtractor.cs index d61456936..f7899bf39 100644 --- a/src/CodeIndex/Indexer/References/Languages/MarkupSchemaReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/MarkupSchemaReferenceExtractor.cs @@ -11,6 +11,7 @@ internal sealed class MarkupState public bool InMarkdownFence { get; set; } public char MarkdownFenceChar { get; set; } public int MarkdownFenceLength { get; set; } + public IReadOnlyDictionary? MarkdownReferenceTargets { get; set; } public bool InHtmlComment { get; set; } public string? HtmlRawTextTag { get; set; } public int GraphQLBraceDepth { get; set; } @@ -110,9 +111,17 @@ internal sealed class MarkupState RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex MarkdownReferenceLinkRegex = new( - @"!?\[[^\]\r\n]+\]\[(?