diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 1bccffccc..1f24cee95 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -638,6 +638,7 @@ Do not add mutable static caches, shared `StringBuilder` instances, reused `Matc |---|---|---| | `accessor` | Accessor declarations when extracted separately from their owning property | Search/filter symbol | | `add` | Dockerfile `ADD` destination paths | Dependency/file-flow search symbol | +| `anchor` | Explicit Markdown HTML anchors, preserving exact ID case and punctuation after HTML entity decoding | Definition target for path-scoped Markdown fragment references | | `annotation` | Annotation declarations or annotation-like language constructs | Metadata/search symbol | | `async_function` | JavaScript/TypeScript async function declarations | Callable definition; participates in callers/callees through reference rows | | `async_generator` | JavaScript/TypeScript async generator declarations | Callable definition; participates in callers/callees through reference rows | @@ -659,7 +660,7 @@ Do not add mutable static caches, shared `StringBuilder` instances, reused `Matc | `file_module` | File-scoped module/package declarations | Namespace-like context symbol | | `function` | Functions, methods, constructors, delegates, tasks, and callable bindings that do not have a narrower kind | Primary callable definition; participates in callers/callees through reference rows | | `generator` | JavaScript/TypeScript generator declarations | Callable definition; participates in callers/callees through reference rows | -| `heading` | Markdown headings and language section markers such as C# regions, Python module docstrings, and JavaScript/TypeScript `@module` docblocks | Outline symbol | +| `heading` | Markdown headings and language section markers such as C# regions, Python module docstrings, and JavaScript/TypeScript `@module` docblocks | Outline symbol; Markdown headings are definition targets for path-scoped fragment references | | `hook` | JavaScript/TypeScript React custom hook bindings | Callable-like search/filter symbol | | `implements` | Razor `@implements` directives | Context/search symbol | | `import` | Imports, using directives, aliases, and package includes | Search/filter symbol | @@ -3784,6 +3785,7 @@ filter、downstream JSON consumer が同じ値を理解できるようにして |---|---|---| | `accessor` | owning property から別 symbol として抽出される accessor declaration | Search/filter symbol | | `add` | Dockerfile `ADD` の destination path | dependency / file-flow search symbol | +| `anchor` | HTML entity の decode 後も ID の大文字小文字と句読点を正確に保持する Markdown 内の明示的な HTML anchor | path に限定した Markdown fragment reference の定義対象 | | `annotation` | annotation declaration または annotation-like な言語構文 | Metadata/search symbol | | `async_function` | JavaScript / TypeScript の async function declaration | Callable definition。reference row 経由で callers/callees に参加 | | `async_generator` | JavaScript / TypeScript の async generator declaration | Callable definition。reference row 経由で callers/callees に参加 | @@ -3805,7 +3807,7 @@ filter、downstream JSON consumer が同じ値を理解できるようにして | `file_module` | file-scoped module / package declaration | Namespace-like context symbol | | `function` | 関数、method、constructor、delegate、task、およびより狭い kind がない callable binding | Primary callable definition。reference row 経由で callers/callees に参加 | | `generator` | JavaScript / TypeScript generator declaration | Callable definition。reference row 経由で callers/callees に参加 | -| `heading` | Markdown heading、C# region、Python module docstring、JavaScript / TypeScript `@module` docblock などの language section marker | Outline symbol | +| `heading` | Markdown heading、C# region、Python module docstring、JavaScript / TypeScript `@module` docblock などの language section marker | Outline symbol。Markdown heading は path に限定した fragment reference の定義対象 | | `hook` | JavaScript / TypeScript React custom hook binding | Callable-like search/filter symbol | | `implements` | Razor `@implements` directive | Context/search symbol | | `import` | import、using directive、alias、package include | Search/filter symbol | diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 65e538490..ddb09d43c 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -2478,7 +2478,7 @@ All indexed languages are searchable through FTS5. Rows with **Symbols = yes** a - Dynamic/declarative graph languages: Crystal, Groovy, Tcl, and Prolog expose conservative declarations, imports, and call relationships. Crystal, Groovy, and Prolog parenthesized calls use the shared extractor; command-style calls are limited to callables declared in the same file. Tcl recognizes command substitutions and common control-command script arguments without treating ordinary `name()` words as calls, while Tcl proc / Prolog predicate bodies preserve caller containers. An index created before this graph contract reports `reference_graph_complete=false` and `graph_data_current=false` with `dynamic_reference_graph_contract_stale`; rerun `cdidx index ` to refresh affected rows before treating absent edges as authoritative. - Scientific and native-extension graphs: Julia, MATLAB, Nim, D, Cython, and Ada emit bounded language-aware import/module, base/type, and call references. Julia macro invocations and Ada procedure-style calls without parentheses are also represented. -- Markdown, JSON/YAML, and CSS: Markdown heading and local-anchor symbols are indexed; JSON/YAML configuration keys are indexed as structural key paths; CSS variables, placeholders, and `@extend` references are indexed. +- Markdown, JSON/YAML, and CSS: Markdown headings and explicit HTML anchors are indexed as definitions, while local and cross-document fragment links are indexed as path-scoped references. Heading slugs use rendered inline text; explicit HTML IDs preserve exact case and punctuation after HTML entity decoding. JSON/YAML configuration keys are indexed as structural key paths; CSS variables, placeholders, and `@extend` references are indexed. - Dockerfile, Assembly, Common Lisp, and Racket: `ARG` build args, labels/PROC/MACRO blocks, package/module forms, definitions, classes/structs, requires, and provides are surfaced as symbols where applicable. - Shell, PowerShell, and Batch: command-style function calls, functions/filters, classes/enums, imports, labels, `goto` / `call` targets, and inline control-flow forms are indexed where the language supports them. - C# and Java: modern C# partial members remain visible to `symbols`, `definition`, and `outline`; Java sealed `permits` lists are recorded as `type_reference` graph edges. @@ -2526,7 +2526,7 @@ entries with the unsupported capability, an explanatory message, and | Shell / PowerShell / Batch / Makefile / CMake / Justfile / MSBuild / Gradle | functions, labels, targets, recipes, tasks, imports where applicable | command-style calls, target dependencies, and control-flow targets | Runtime command construction is not resolved. | | Solution / application manifest | solution projects and manifest identity/settings | solution project references; application manifests are symbol-only | `.sln` project paths are graph edges for repository structure; use `symbols --lang app_manifest` for Windows manifest metadata. | | SQL / Terraform / Dockerfile | statements/resources/stages/labels | table/resource/stage references, Dockerfile stage dependencies, Terraform dotted refs | SQL hotspot grouping defaults to statements; Dockerfile `COPY --from=` follows named stages. | -| Markdown / HTML / CSS / Sass / Stylus / XML / XAML / GraphQL / Protobuf | headings, anchors, selectors, UI elements, generic XML element/attribute paths, schema types/messages where supported | links/assets/components, local anchors, CSS/Sass/Stylus imports, variables, mixins/functions, XAML resources/bindings/handlers, schema references where supported | Generic non-XAML XML emits bounded structural symbols; use `search` for prose and generated markup. | +| Markdown / HTML / CSS / Sass / Stylus / XML / XAML / GraphQL / Protobuf | headings, explicit anchors, selectors, UI elements, generic XML element/attribute paths, schema types/messages where supported | links/assets/components, path-scoped local and cross-document fragments, CSS/Sass/Stylus imports, variables, mixins/functions, XAML resources/bindings/handlers, schema references where supported | Markdown fragment references resolve only against headings or explicit anchors in the linked document. Generic non-XAML XML emits bounded structural symbols; use `search` for prose and generated markup. | | Dependency manifests / lockfiles | none | none | Use `--lang dependency_manifest` or `--lang dependency_lock` for dependency/security audits. | | Other indexed text formats | file/chunk search only unless `languages` reports symbols | no graph unless `languages` reports support | `cdidx search "literal" --lang yaml` is the reliable fallback. | @@ -5568,7 +5568,7 @@ indexing はファイル単位の SQLite transaction を commit します。長 - 動的・宣言型言語の graph 対応: Crystal、Groovy、Tcl、Prolog は保守的な宣言、import、call relationship を公開します。Crystal、Groovy、Prolog の括弧付き call は共通 extractor を使い、command-style call は同一ファイルで宣言済みの callable に限定します。Tcl は通常の `name()` word を call とみなさず、command substitution と主要な制御 command の script 引数を認識し、Tcl proc / Prolog predicate の本体では caller container を保持します。 この graph contract より前に作成された index は `dynamic_reference_graph_contract_stale` とともに `reference_graph_complete=false`、`graph_data_current=false` を報告します。欠落 edge を authoritative とみなす前に `cdidx index ` を再実行して対象 row を更新してください。 - 科学技術・ネイティブ拡張言語のグラフ: Julia、MATLAB、Nim、D、Cython、Ada は、言語構文に応じた import/module、基底型/type、call 参照を上限付きで出力します。Julia の macro invocation と、括弧を伴わない Ada の procedure call も記録します。 -- Markdown、JSON/YAML、CSS: Markdown の heading / local anchor、JSON/YAML の configuration key path、CSS の variable、placeholder、`@extend` をシンボルとして扱います。 +- Markdown、JSON/YAML、CSS: Markdown の heading と明示的な HTML anchor は定義として、同一文書・文書間の fragment link は対象 path に限定した参照として索引します。heading slug は表示される inline text から作り、明示的な HTML ID は HTML entity の decode 後も大文字小文字と句読点を正確に保持します。JSON/YAML の configuration key path、CSS の variable、placeholder、`@extend` もシンボルとして扱います。 - Dockerfile、Assembly、Common Lisp、Racket: `ARG` build arg、label、PROC/MACRO、package/module form、definition、class/struct、require/provide を必要に応じて表面化します。 - Shell、PowerShell、Batch: command-style function call、function/filter、class/enum、import、label、`goto` / `call` target、inline control-flow を言語仕様に合わせて索引します。 - C# と Java: C# の近年の partial member は `symbols`、`definition`、`outline` から見えます。Java の sealed `permits` list は `type_reference` graph edge として記録します。 @@ -5610,7 +5610,7 @@ indexing はファイル単位の SQLite transaction を commit します。長 | Shell / PowerShell / Batch / Makefile / CMake / Justfile / MSBuild / Gradle | function、label、target、recipe、task、対応言語の import | command-style call、target dependency、control-flow target | runtime で組み立てられる command は解決しません。 | | ソリューション / アプリケーションマニフェスト | solution project、manifest identity / setting | `.sln` の project reference。manifest は symbol-only | `.sln` の project path はリポジトリ構造の graph edge です。Windows manifest metadata は `symbols --lang app_manifest` で確認できます。 | | SQL / Terraform / Dockerfile | statement/resource/stage/label | table/resource/stage reference、Dockerfile stage dependency、Terraform dotted refs | SQL hotspot grouping は既定で statement、Dockerfile `COPY --from=` は named stage を追跡します。 | -| Markdown / HTML / CSS / Sass / Stylus / XML / XAML / GraphQL / Protobuf | heading、anchor、selector、UI element、汎用 XML の要素・属性パス、対応 schema type/message | link/asset/component、local anchor、CSS/Sass/Stylus の import・variable・mixin/function、XAML resource / binding / handler、対応 schema reference | 汎用の非 XAML XML は上限付きの構造シンボルを出力します。prose や generated markup には `search` を使ってください。 | +| Markdown / HTML / CSS / Sass / Stylus / XML / XAML / GraphQL / Protobuf | heading、明示的な anchor、selector、UI element、汎用 XML の要素・属性パス、対応 schema type/message | link/asset/component、path に限定した同一文書・文書間の fragment、CSS/Sass/Stylus の import・variable・mixin/function、XAML resource / binding / handler、対応 schema reference | Markdown の fragment reference は、リンク先文書内の heading または明示的な anchor にのみ解決します。汎用の非 XAML XML は上限付きの構造シンボルを出力します。prose や generated markup には `search` を使ってください。 | | Dependency manifest / lockfile | なし | なし | dependency / security audit には `--lang dependency_manifest` または `--lang dependency_lock` を使います。 | | その他の indexed text format | `languages` が symbol 対応を示す場合を除き file/chunk search のみ | `languages` が graph 対応を示す場合を除きなし | `cdidx search "literal" --lang yaml` が信頼できる fallback です。 | diff --git a/changelog.d/unreleased/4846.fixed.md b/changelog.d/unreleased/4846.fixed.md new file mode 100644 index 000000000..93f5f1afd --- /dev/null +++ b/changelog.d/unreleased/4846.fixed.md @@ -0,0 +1,35 @@ +--- +category: fixed +issues: + - 4846 +affected: + - src/CodeIndex/Database/DbContext.ConnectionFunctions.cs + - src/CodeIndex/Database/DbContext.SchemaMetadata.cs + - src/CodeIndex/Indexer/MarkdownAnchorIdentity.cs + - src/CodeIndex/Models/SymbolKindCatalog.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Markup.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Contracts.cs + - src/CodeIndex/Indexer/References/Languages/MarkupSchemaReferenceExtractor.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.CoreReferenceLoop.cs + - src/CodeIndex/Indexer/References/ReferenceExtractor.ReferenceRecords.cs + - src/CodeIndex/Database/DbSymbolReader.Definitions.cs + - src/CodeIndex/Database/DbSymbolReader.Search.cs + - src/CodeIndex/Database/DbWriter.FoldBackfill.cs + - src/CodeIndex/Database/DbWriter.References.cs + - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs + - tests/CodeIndex.Tests/DbReaderFileDependencyTests.cs + - tests/CodeIndex.Tests/DatabaseTests.cs + - tests/CodeIndex.Tests/ReferenceExtractorTests.cs + - tests/CodeIndex.Tests/SymbolExtractorTests.cs + - tests/CodeIndex.Tests/IndexCommandRunnerUpdateTests.cs + - USER_GUIDE.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Markdown fragment links now resolve to path-scoped heading and explicit-anchor definitions (#4846)** — local and cross-document links use rendered heading text for normalized slugs, preserve exact case and punctuation for entity-decoded explicit HTML IDs, refresh references when only the target document changes, keep duplicate heading slugs distinct, and no longer create false dependency edges from equal heading names in unrelated documents. This completes the Markdown definition follow-up to #4113. + +## 日本語 + +- **Markdown の fragment link が path に限定された heading と明示的な anchor の定義へ解決されるようになりました (#4846)** — 同一文書・文書間の link は表示される heading text から正規化した slug を作り、entity を decode した明示的な HTML ID の大文字小文字と句読点を正確に保持し、対象文書だけが変わった場合も参照を更新し、重複する heading slug を区別します。また、無関係な文書の同名 heading による誤った依存 edge を作らなくなりました。これは #4113 に続く Markdown definition の修正です。 diff --git a/src/CodeIndex/Database/DbContext.ConnectionFunctions.cs b/src/CodeIndex/Database/DbContext.ConnectionFunctions.cs index 1e4221bdf..b36d84bc9 100644 --- a/src/CodeIndex/Database/DbContext.ConnectionFunctions.cs +++ b/src/CodeIndex/Database/DbContext.ConnectionFunctions.cs @@ -31,6 +31,11 @@ internal static void RegisterConnectionFunctions(SqliteConnection connection) connection.CreateFunction( "markdown_resolve_path", (string? sourcePath, string? targetPath) => DbReader.ResolveMarkdownDependencyPath(sourcePath, targetPath)); + connection.CreateFunction( + "markdown_normalize_fragment", + (string? fragment) => fragment == null + ? null + : MarkdownAnchorIdentity.NormalizeHeadingFragment(fragment)); connection.CreateFunction( "python_import_resolves", (string? sourcePath, string? targetPath, string? referenceName, string? referenceKind, string? context, long? columnNumber, string? signature) => diff --git a/src/CodeIndex/Database/DbContext.SchemaMetadata.cs b/src/CodeIndex/Database/DbContext.SchemaMetadata.cs index 2c776ba92..63c46039b 100644 --- a/src/CodeIndex/Database/DbContext.SchemaMetadata.cs +++ b/src/CodeIndex/Database/DbContext.SchemaMetadata.cs @@ -62,13 +62,15 @@ public static string BuildIncompleteHotspotFamilyMarkerFingerprint(string? finge public const string SqlGraphContractVersionMetaKey = "sql_graph_contract_version"; public const int HdlGraphContractVersion = 1; public const string HdlGraphContractVersionMetaKey = "hdl_graph_contract_version"; - // Version 4 (#4845) invalidates dependency-lock candidates written before resolution - // became file-local. Version 3 (#4825) previously invalidated C# type-reference - // candidates written before type-like kind and compatible generic-arity constraints. - // バージョン 4 (#4845) では、解決を file 内に限定する前に書かれた dependency-lock - // candidate を無効化する。バージョン 3 (#4825) では、それ以前に C# 型参照の - // type-like kind / 互換 generic arity 制約より前に書かれた candidate を無効化した。 - public const int ReferenceIdentityContractVersion = 4; + // Version 5 (#4846) invalidates name-only Markdown candidates written before + // fragment resolution became document/path-scoped. Version 4 (#4845) previously + // invalidated dependency-lock candidates written before resolution became file-local, + // and version 3 (#4825) constrained C# type-reference candidates by kind and arity. + // バージョン 5 (#4846) では、Markdown fragment の解決を document/path 内に限定する + // 前に書かれた name-only candidate を無効化する。バージョン 4 (#4845) では、それ以前の + // dependency-lock candidate を file 内に限定し、バージョン 3 (#4825) では C# 型参照を + // kind と arity で制約した。 + public const int ReferenceIdentityContractVersion = 5; public const string ReferenceIdentityContractVersionMetaKey = "reference_identity_contract_version"; public static string GetDynamicReferenceGraphContractVersionMetaKey(string lang) => $"dynamic_reference_graph_contract_version_{lang}"; diff --git a/src/CodeIndex/Database/DbSymbolReader.Definitions.cs b/src/CodeIndex/Database/DbSymbolReader.Definitions.cs index 26ba01a4f..188d8cb68 100644 --- a/src/CodeIndex/Database/DbSymbolReader.Definitions.cs +++ b/src/CodeIndex/Database/DbSymbolReader.Definitions.cs @@ -328,6 +328,12 @@ FROM symbols s var qualifiedSymbolClause = SqlNameResolver.HasQualifier(normalizedQuery) ? BuildQualifiedSymbolMatchSql("query", _foldReady) : null; + var markdownAnchorExactClause = _symbolColumns.Contains("name_folded") + ? "(f.lang = 'markdown' AND ((s.kind = 'heading' AND s.name_folded = @queryMarkdownHeading) OR (s.kind = 'anchor' AND s.name_folded = @queryMarkdownExplicitAnchor COLLATE BINARY)))" + : "0"; + var markdownAnchorLikeClause = _symbolColumns.Contains("name_folded") + ? " OR (f.lang = 'markdown' AND ((s.kind = 'heading' AND s.name_folded LIKE @queryMarkdownHeadingLike ESCAPE '\\') OR (s.kind = 'anchor' AND instr(s.name_folded, @queryMarkdownExplicitAnchor) > 0)))" + : string.Empty; sql += exact ? rustQualifiedParts.QualifiedPath != null ? _foldReady @@ -335,12 +341,12 @@ FROM symbols s : " AND ((s.container_qualified_name = @queryRustContainer COLLATE NOCASE OR s.container_name = @queryRustContainer COLLATE NOCASE) AND s.name = @queryRustLeaf COLLATE NOCASE)" : _foldReady ? allowLeafFallback - ? $" AND ({BuildPersistedFoldedNameMatchSql("s.name_folded", "@query")} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded) OR sql_leaf_name_folded(s.name) = @queryLeafFolded)))" - : $" AND ({BuildPersistedFoldedNameMatchSql("s.name_folded", "@query")} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" + ? $" AND ({BuildPersistedFoldedNameMatchSql("s.name_folded", "@query")} OR {markdownAnchorExactClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded) OR sql_leaf_name_folded(s.name) = @queryLeafFolded)))" + : $" AND ({BuildPersistedFoldedNameMatchSql("s.name_folded", "@query")} OR {markdownAnchorExactClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name_folded(s.name) = @queryNormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" : allowLeafFallback - ? " AND (s.name = @query COLLATE NOCASE OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name(s.name) = @queryNormalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @queryLeaf COLLATE NOCASE)))" - : $" AND (s.name = @query COLLATE NOCASE OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name(s.name) = @queryNormalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" - : $" AND (s.name LIKE @query ESCAPE '\\' OR (f.lang = 'sql' AND sql_normalize_name(s.name) LIKE @queryNormalizedLike ESCAPE '\\'){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; + ? $" AND (s.name = @query COLLATE NOCASE OR {markdownAnchorExactClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name(s.name) = @queryNormalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @queryLeaf COLLATE NOCASE)))" + : $" AND (s.name = @query COLLATE NOCASE OR {markdownAnchorExactClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @querySegmentCount AND sql_normalize_name(s.name) = @queryNormalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" + : $" AND (s.name LIKE @query ESCAPE '\\'{markdownAnchorLikeClause} OR (f.lang = 'sql' AND sql_normalize_name(s.name) LIKE @queryNormalizedLike ESCAPE '\\'){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; } if (kind != null) sql += " AND s.kind = @kind"; @@ -379,6 +385,13 @@ FROM chunks c SqliteCommandPolicy.Add(cmd, "@queryLeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(normalizedQuery)) ?? SqlNameResolver.GetLeafName(normalizedQuery)); SqliteCommandPolicy.Add(cmd, "@querySegmentCount", SqlNameResolver.GetSegmentCount(normalizedQuery)); SqliteCommandPolicy.Add(cmd, "@queryNormalizedLike", $"%{EscapeLikeQuery(SqlNameResolver.NormalizeQualifiedName(normalizedQuery))}%"); + if (_symbolColumns.Contains("name_folded")) + { + var markdownHeadingIdentity = MarkdownAnchorIdentity.NormalizeHeadingFragment(normalizedQuery); + SqliteCommandPolicy.Add(cmd, "@queryMarkdownHeading", markdownHeadingIdentity); + SqliteCommandPolicy.Add(cmd, "@queryMarkdownHeadingLike", $"%{EscapeLikeQuery(markdownHeadingIdentity)}%"); + SqliteCommandPolicy.Add(cmd, "@queryMarkdownExplicitAnchor", MarkdownAnchorIdentity.NormalizeExplicitAnchorDefinition(normalizedQuery)); + } if (SqlNameResolver.HasQualifier(normalizedQuery)) AddQualifiedSymbolQueryParameters(cmd, "query", normalizedQuery); if (rustQualifiedParts.QualifiedPath != null) diff --git a/src/CodeIndex/Database/DbSymbolReader.Search.cs b/src/CodeIndex/Database/DbSymbolReader.Search.cs index 707420db4..0305afd3a 100644 --- a/src/CodeIndex/Database/DbSymbolReader.Search.cs +++ b/src/CodeIndex/Database/DbSymbolReader.Search.cs @@ -193,8 +193,10 @@ private static void AddQualifiedGraphQueryParameters(SqliteCommand cmd, string q public int CountSearchSymbols(IReadOnlyList? queries, int limit = 20, string? kind = null, string? lang = null, IReadOnlyList? pathPatterns = null, IReadOnlyList? excludePathPatterns = null, bool excludeTests = false, DateTime? since = null, bool exact = false, IReadOnlyList? visibilityFilters = null, IReadOnlyList? excludeVisibilityFilters = null) { + lang = DbReader.NormalizeQueryLanguage(lang); var validQueries = NormalizeSymbolSearchQueries(queries, lang, exact); - if (validQueries != null && validQueries.Count > 1) + if (validQueries != null + && (validQueries.Count > 1 || string.Equals(lang, "markdown", StringComparison.Ordinal))) return SearchSymbols(validQueries, limit, kind, lang, pathPatterns, excludePathPatterns, excludeTests, since, exact, visibilityFilters, excludeVisibilityFilters).Count; using var cmd = _conn.CreateCommand(); @@ -318,6 +320,9 @@ FROM symbols s var rustQualifiedExact = ShouldPreserveRustQualifiedExactQuery(queryValue, lang, exact); var rustQualifiedParts = rustQualifiedExact ? NormalizeRustQualifiedExactQueryParts(queryValue) : default; var allowLeafFallback = !SqlNameResolver.HasQualifier(queryValue); + var markdownAnchorClause = _symbolColumns.Contains("name_folded") + ? $"(f.lang = 'markdown' AND ((s.kind = 'heading' AND s.name_folded = @query{idx}MarkdownHeading) OR (s.kind = 'anchor' AND s.name_folded = @query{idx}MarkdownExplicitAnchor COLLATE BINARY)))" + : "0"; var qualifiedSymbolClause = SqlNameResolver.HasQualifier(queryValue) ? BuildQualifiedSymbolMatchSql($"query{idx}", _foldReady) : null; @@ -333,18 +338,21 @@ FROM symbols s : $"((s.container_qualified_name = @query{idx}RustContainer COLLATE NOCASE OR s.container_name = @query{idx}RustContainer COLLATE NOCASE) AND s.name = @query{idx}RustLeaf COLLATE NOCASE)"; return _foldReady ? allowLeafFallback - ? $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query{idx}LeafFolded)))" - : $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" + ? $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query{idx}LeafFolded)))" + : $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" : allowLeafFallback - ? $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @query{idx}Leaf COLLATE NOCASE)))" - : $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; + ? $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @query{idx}Leaf COLLATE NOCASE)))" + : $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; })) : string.Join(" OR ", effectiveQueries.Select((queryValue, idx) => { var qualifiedSymbolClause = SqlNameResolver.HasQualifier(queryValue) ? BuildQualifiedSymbolMatchSql($"query{idx}", _foldReady) : null; - return $"(s.name LIKE @query{idx} ESCAPE '\\' OR (f.lang = 'sql' AND sql_normalize_name(s.name) LIKE @query{idx}NormalizedLike ESCAPE '\\'){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; + var markdownAnchorLikeClause = _symbolColumns.Contains("name_folded") + ? $" OR (f.lang = 'markdown' AND ((s.kind = 'heading' AND s.name_folded LIKE @query{idx}MarkdownHeadingLike ESCAPE '\\') OR (s.kind = 'anchor' AND instr(s.name_folded, @query{idx}MarkdownExplicitAnchor) > 0)))" + : string.Empty; + return $"(s.name LIKE @query{idx} ESCAPE '\\'{markdownAnchorLikeClause} OR (f.lang = 'sql' AND sql_normalize_name(s.name) LIKE @query{idx}NormalizedLike ESCAPE '\\'){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; })); sql += $" AND ({orClauses})"; } @@ -378,6 +386,13 @@ FROM symbols s SqliteCommandPolicy.Add(cmd, $"@query{i}LeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(value)) ?? SqlNameResolver.GetLeafName(value)); SqliteCommandPolicy.Add(cmd, $"@query{i}SegmentCount", SqlNameResolver.GetSegmentCount(value)); SqliteCommandPolicy.Add(cmd, $"@query{i}NormalizedLike", $"%{EscapeLikeQuery(SqlNameResolver.NormalizeQualifiedName(value))}%"); + if (_symbolColumns.Contains("name_folded")) + { + var markdownHeadingIdentity = MarkdownAnchorIdentity.NormalizeHeadingFragment(value); + SqliteCommandPolicy.Add(cmd, $"@query{i}MarkdownHeading", markdownHeadingIdentity); + SqliteCommandPolicy.Add(cmd, $"@query{i}MarkdownHeadingLike", $"%{EscapeLikeQuery(markdownHeadingIdentity)}%"); + SqliteCommandPolicy.Add(cmd, $"@query{i}MarkdownExplicitAnchor", MarkdownAnchorIdentity.NormalizeExplicitAnchorDefinition(value)); + } if (SqlNameResolver.HasQualifier(value)) AddQualifiedSymbolQueryParameters(cmd, $"query{i}", value); var swiftBacktickAlias = ComputeSwiftBacktickAlias(value, lang); @@ -599,6 +614,9 @@ FROM symbols s var rustQualifiedExact = ShouldPreserveRustQualifiedExactQuery(queryValue, lang, exact); var rustQualifiedParts = rustQualifiedExact ? NormalizeRustQualifiedExactQueryParts(queryValue) : default; var allowLeafFallback = !SqlNameResolver.HasQualifier(queryValue); + var markdownAnchorClause = _symbolColumns.Contains("name_folded") + ? $"(f.lang = 'markdown' AND ((s.kind = 'heading' AND s.name_folded = @query{idx}MarkdownHeading) OR (s.kind = 'anchor' AND s.name_folded = @query{idx}MarkdownExplicitAnchor COLLATE BINARY)))" + : "0"; var qualifiedSymbolClause = SqlNameResolver.HasQualifier(queryValue) ? BuildQualifiedSymbolMatchSql($"query{idx}", _foldReady) : null; @@ -614,18 +632,21 @@ FROM symbols s : $"((s.container_qualified_name = @query{idx}RustContainer COLLATE NOCASE OR s.container_name = @query{idx}RustContainer COLLATE NOCASE) AND s.name = @query{idx}RustLeaf COLLATE NOCASE)"; return _foldReady ? allowLeafFallback - ? $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query{idx}LeafFolded)))" - : $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" + ? $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded) OR sql_leaf_name_folded(s.name) = @query{idx}LeafFolded)))" + : $"({BuildPersistedFoldedNameMatchSql("s.name_folded", $"@query{idx}")}{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name_folded(s.name) = @query{idx}NormalizedFolded){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})" : allowLeafFallback - ? $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @query{idx}Leaf COLLATE NOCASE)))" - : $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; + ? $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND ((sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE) OR sql_leaf_name(s.name) = @query{idx}Leaf COLLATE NOCASE)))" + : $"(s.name = @query{idx} COLLATE NOCASE{swiftBacktickClause} OR {markdownAnchorClause} OR (f.lang = 'sql' AND sql_segment_count(s.name) = @query{idx}SegmentCount AND sql_normalize_name(s.name) = @query{idx}Normalized COLLATE NOCASE){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; })) : string.Join(" OR ", effectiveQueries.Select((queryValue, idx) => { var qualifiedSymbolClause = SqlNameResolver.HasQualifier(queryValue) ? BuildQualifiedSymbolMatchSql($"query{idx}", _foldReady) : null; - return $"(s.name LIKE @query{idx} ESCAPE '\\' OR (f.lang = 'sql' AND sql_normalize_name(s.name) LIKE @query{idx}NormalizedLike ESCAPE '\\'){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; + var markdownAnchorLikeClause = _symbolColumns.Contains("name_folded") + ? $" OR (f.lang = 'markdown' AND ((s.kind = 'heading' AND s.name_folded LIKE @query{idx}MarkdownHeadingLike ESCAPE '\\') OR (s.kind = 'anchor' AND instr(s.name_folded, @query{idx}MarkdownExplicitAnchor) > 0)))" + : string.Empty; + return $"(s.name LIKE @query{idx} ESCAPE '\\'{markdownAnchorLikeClause} OR (f.lang = 'sql' AND sql_normalize_name(s.name) LIKE @query{idx}NormalizedLike ESCAPE '\\'){(qualifiedSymbolClause != null ? $" OR {qualifiedSymbolClause}" : string.Empty)})"; })); sql += $" AND ({orClauses})"; } @@ -679,6 +700,13 @@ FROM symbols s SqliteCommandPolicy.Add(cmd, $"@query{idx}LeafFolded", NameFold.Fold(SqlNameResolver.GetLeafName(effectiveQueries[idx])) ?? SqlNameResolver.GetLeafName(effectiveQueries[idx])); SqliteCommandPolicy.Add(cmd, $"@query{idx}SegmentCount", SqlNameResolver.GetSegmentCount(effectiveQueries[idx])); SqliteCommandPolicy.Add(cmd, $"@query{idx}NormalizedLike", $"%{EscapeLikeQuery(SqlNameResolver.NormalizeQualifiedName(effectiveQueries[idx]))}%"); + if (_symbolColumns.Contains("name_folded")) + { + var markdownHeadingIdentity = MarkdownAnchorIdentity.NormalizeHeadingFragment(effectiveQueries[idx]); + SqliteCommandPolicy.Add(cmd, $"@query{idx}MarkdownHeading", markdownHeadingIdentity); + SqliteCommandPolicy.Add(cmd, $"@query{idx}MarkdownHeadingLike", $"%{EscapeLikeQuery(markdownHeadingIdentity)}%"); + SqliteCommandPolicy.Add(cmd, $"@query{idx}MarkdownExplicitAnchor", MarkdownAnchorIdentity.NormalizeExplicitAnchorDefinition(effectiveQueries[idx])); + } if (SqlNameResolver.HasQualifier(effectiveQueries[idx])) AddQualifiedSymbolQueryParameters(cmd, $"query{idx}", effectiveQueries[idx]); if (rustQualifiedParts.QualifiedPath != null) diff --git a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs index 9f7d00993..53ba324a5 100644 --- a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs +++ b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs @@ -94,9 +94,10 @@ private bool AllFoldedColumnsBackfilledCore( public bool AllFoldedColumnValuesMatchCurrentFold() { + var markdownSymbolIdentityFolds = BuildMarkdownSymbolIdentityFoldMap(); var symbols = RentCommand( """ - SELECT s.name, s.name_folded, f.lang + SELECT s.id, s.name, s.name_folded, f.lang, s.kind FROM symbols s JOIN files f ON f.id = s.file_id WHERE s.name IS NOT NULL @@ -107,10 +108,13 @@ WHERE s.name IS NOT NULL using var reader = symbols.ExecuteTrackedReader(); while (reader.TrackedRead()) { - var expected = DbReader.FoldNameForLanguage( - reader.GetString(0), - reader.IsDBNull(2) ? null : reader.GetString(2)); - var actual = reader.IsDBNull(1) ? null : reader.GetString(1); + var expected = FoldPersistedSymbolName( + reader.GetInt64(0), + reader.GetString(1), + reader.IsDBNull(3) ? null : reader.GetString(3), + reader.GetString(4), + markdownSymbolIdentityFolds); + var actual = reader.IsDBNull(2) ? null : reader.GetString(2); if (!string.Equals(actual, expected, StringComparison.Ordinal)) return false; } @@ -124,7 +128,7 @@ WHERE s.name IS NOT NULL @" SELECT r.symbol_name, r.symbol_name_folded, r.container_name, r.container_name_folded, - f.lang + f.lang, r.reference_kind FROM symbol_references r JOIN files f ON f.id = r.file_id WHERE r.symbol_name IS NOT NULL OR r.container_name IS NOT NULL", @@ -136,9 +140,10 @@ FROM symbol_references r { if (!reader.IsDBNull(0)) { - var expected = DbReader.FoldNameForLanguage( + var expected = FoldPersistedReferenceName( reader.GetString(0), - reader.IsDBNull(4) ? null : reader.GetString(4)); + reader.IsDBNull(4) ? null : reader.GetString(4), + reader.GetString(5)); var actual = reader.IsDBNull(1) ? null : reader.GetString(1); if (!string.Equals(actual, expected, StringComparison.Ordinal)) return false; @@ -372,18 +377,19 @@ private int BackfillSymbolFoldedRows(bool rewriteAll, CancellationToken cancella if (phase == "references") return 0; + var markdownSymbolIdentityFolds = BuildMarkdownSymbolIdentityFoldMap(); var lastSymbolId = rewriteAll ? GetFoldBackfillCheckpoint(FoldBackfillLastSymbolIdMetaKey) : 0; - var rows = new List<(long Id, string Name, string? Lang)>(); + var rows = new List<(long Id, string Name, string? Lang, string Kind)>(); var selectSql = rewriteAll ? """ - SELECT s.id, s.name, f.lang + SELECT s.id, s.name, f.lang, s.kind FROM symbols s JOIN files f ON f.id = s.file_id WHERE s.name IS NOT NULL AND s.id > @lastSymbolId ORDER BY s.id """ : """ - SELECT s.id, s.name, f.lang + SELECT s.id, s.name, f.lang, s.kind FROM symbols s JOIN files f ON f.id = s.file_id WHERE s.name IS NOT NULL AND s.name_folded IS NULL @@ -404,7 +410,8 @@ WHERE s.name IS NOT NULL AND s.name_folded IS NULL rows.Add(( reader.GetInt64(0), reader.GetString(1), - reader.IsDBNull(2) ? null : reader.GetString(2))); + reader.IsDBNull(2) ? null : reader.GetString(2), + reader.GetString(3))); } } finally @@ -429,7 +436,12 @@ WHERE s.name IS NOT NULL AND s.name_folded IS NULL foreach (var row in rows) { cancellationToken.ThrowIfCancellationRequested(); - pFolded.Value = DbReader.FoldNameForLanguage(row.Name, row.Lang); + pFolded.Value = FoldPersistedSymbolName( + row.Id, + row.Name, + row.Lang, + row.Kind, + markdownSymbolIdentityFolds); pId.Value = row.Id; update.ExecuteNonQuery(); if (rewriteAll) @@ -445,13 +457,88 @@ WHERE s.name IS NOT NULL AND s.name_folded IS NULL return rows.Count; } + private Dictionary BuildMarkdownSymbolIdentityFoldMap() + { + var identities = new Dictionary(); + var usedHeadingIdentitiesByFile = new Dictionary>(); + var select = RentCommand( + """ + SELECT s.id, s.file_id, s.kind, s.name + FROM symbols s + JOIN files f ON f.id = s.file_id + WHERE f.lang = 'markdown' + AND s.kind IN ('heading', 'anchor') + AND s.name IS NOT NULL + ORDER BY s.file_id, COALESCE(s.start_line, s.line), s.id + """, + static _ => { }); + try + { + using var reader = select.ExecuteTrackedReader(); + while (reader.TrackedRead()) + { + var symbolId = reader.GetInt64(0); + var fileId = reader.GetInt64(1); + var kind = reader.GetString(2); + var name = reader.GetString(3); + if (kind == "heading") + { + if (!usedHeadingIdentitiesByFile.TryGetValue(fileId, out var usedHeadingIdentities)) + { + usedHeadingIdentities = new HashSet(StringComparer.Ordinal); + usedHeadingIdentitiesByFile.Add(fileId, usedHeadingIdentities); + } + + identities.Add( + symbolId, + MarkdownAnchorIdentity.CreateUniqueHeadingIdentity(name, usedHeadingIdentities)); + } + else + { + identities.Add(symbolId, MarkdownAnchorIdentity.NormalizeExplicitAnchorDefinition(name)); + } + } + } + finally + { + ReleaseCommand(select); + } + + return identities; + } + + private static string FoldPersistedSymbolName( + long symbolId, + string name, + string? lang, + string kind, + IReadOnlyDictionary markdownSymbolIdentityFolds) + { + if (lang == "markdown" + && (kind == "heading" || kind == "anchor") + && markdownSymbolIdentityFolds.TryGetValue(symbolId, out var identity)) + { + return identity; + } + + return DbReader.FoldNameForLanguage(name, lang); + } + + private static string FoldPersistedReferenceName(string name, string? lang, string referenceKind) + { + if (lang == "markdown" && referenceKind == "reference") + return MarkdownAnchorIdentity.NormalizeHeadingFragment(name); + + return DbReader.FoldNameForLanguage(name, lang); + } + private int BackfillReferenceFoldedRows(bool rewriteAll, CancellationToken cancellationToken) { var lastReferenceId = rewriteAll ? GetFoldBackfillCheckpoint(FoldBackfillLastReferenceIdMetaKey) : 0; - var rows = new List<(long Id, string? SymbolName, string? ContainerName, string? Lang)>(); + var rows = new List<(long Id, string? SymbolName, string? ContainerName, string? Lang, string ReferenceKind)>(); var selectSql = rewriteAll ? """ - SELECT r.id, r.symbol_name, r.container_name, f.lang + SELECT r.id, r.symbol_name, r.container_name, f.lang, r.reference_kind FROM symbol_references r JOIN files f ON f.id = r.file_id WHERE r.id > @lastReferenceId @@ -459,7 +546,7 @@ WHERE r.id > @lastReferenceId ORDER BY r.id """ : """ - SELECT r.id, r.symbol_name, r.container_name, f.lang + SELECT r.id, r.symbol_name, r.container_name, f.lang, r.reference_kind FROM symbol_references r JOIN files f ON f.id = r.file_id WHERE (r.symbol_name IS NOT NULL AND r.symbol_name_folded IS NULL) @@ -482,7 +569,8 @@ FROM symbol_references r reader.GetInt64(0), reader.IsDBNull(1) ? null : reader.GetString(1), reader.IsDBNull(2) ? null : reader.GetString(2), - reader.IsDBNull(3) ? null : reader.GetString(3))); + reader.IsDBNull(3) ? null : reader.GetString(3), + reader.GetString(4))); } } finally @@ -514,7 +602,7 @@ FROM symbol_references r cancellationToken.ThrowIfCancellationRequested(); pSymbolNameFolded.Value = row.SymbolName == null ? DBNull.Value - : DbReader.FoldNameForLanguage(row.SymbolName, row.Lang); + : FoldPersistedReferenceName(row.SymbolName, row.Lang, row.ReferenceKind); pContainerNameFolded.Value = row.ContainerName == null ? DBNull.Value : DbReader.FoldNameForLanguage(row.ContainerName, row.Lang); diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index 1f206d70d..4ba13cce6 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -157,7 +157,7 @@ private static string BuildScopedReferenceCandidatesSql() const string fullInstantiateNamePredicateSql = "AND s.name_folded IS NOT NULL"; const string fullCSharpTypeSymbolSourceSql = "FROM symbols AS type_symbol"; const string fullCSharpTypeNamePredicateSql = "AND type_symbol.name_folded IS NOT NULL"; - const int expectedReferenceSourceCount = 12; + const int expectedReferenceSourceCount = 14; if (CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullDeleteSql) != 1 || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullReferenceSourceSql) @@ -247,6 +247,22 @@ CROSS JOIN symbol_references AS r INDEXED BY idx_symbol_refs_symbol_name_folded source_file.lang = dirty_name.lang OR (source_file.lang = 'ambiguous_m' AND dirty_name.lang IN ('matlab', 'objc')) ); + + -- Explicit Markdown anchor identities preserve case and punctuation, while the + -- indexed reference fold also carries the equivalent heading slug. Use that slug + -- for the indexed seek, then require the exact raw identity. + -- Markdown の明示 anchor は大文字小文字と句読点を保持するため、heading slug を + -- index seek に使った後で raw identity の完全一致を要求する。 + INSERT OR IGNORE INTO temp.{ReferenceGraphDirtyReferencesTable}(reference_id) + SELECT r.id + FROM temp.{ReferenceGraphDirtyNamesTable} AS dirty_name + CROSS JOIN symbol_references AS r INDEXED BY idx_symbol_refs_symbol_name_folded + JOIN files AS source_file ON source_file.id = r.file_id + WHERE dirty_name.lang = 'markdown' + AND source_file.lang = 'markdown' + AND r.reference_kind = 'reference' + AND r.symbol_name_folded = markdown_normalize_fragment(dirty_name.name_folded) + AND r.symbol_name = dirty_name.name_folded COLLATE BINARY; """; private const string MaterializeReferenceGraphLookupNamesSql = $""" diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 86151ae26..2cef55151 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -412,6 +412,38 @@ HAVING COUNT(DISTINCT target_file.path || char(31) || private static string RefreshReferenceCandidatesSql => $""" DELETE FROM symbol_reference_candidates; + INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) + SELECT r.id, target.id, 0 + FROM symbol_references AS r + JOIN files AS source_file ON source_file.id = r.file_id + JOIN symbols AS target + ON target.file_id = source_file.id + AND ( + (target.kind = 'heading' AND target.name_folded = r.symbol_name_folded) + OR (target.kind = 'anchor' AND target.name_folded = r.symbol_name COLLATE BINARY) + ) + WHERE source_file.lang = 'markdown' + AND r.reference_kind = 'reference' + AND r.target_qualifier IS NULL; + + INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) + SELECT r.id, target.id, 0 + FROM symbol_references AS r + JOIN files AS source_file ON source_file.id = r.file_id + JOIN files AS target_file + ON target_file.lang = 'markdown' + AND target_file.path = + markdown_resolve_path(source_file.path, r.target_qualifier) + JOIN symbols AS target + ON target.file_id = target_file.id + AND ( + (target.kind = 'heading' AND target.name_folded = r.symbol_name_folded) + OR (target.kind = 'anchor' AND target.name_folded = r.symbol_name COLLATE BINARY) + ) + WHERE source_file.lang = 'markdown' + AND r.reference_kind = 'reference' + AND r.target_qualifier IS NOT NULL; + INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) SELECT r.id, s.id, 0 FROM symbol_references AS r @@ -445,6 +477,7 @@ THEN r.symbol_name_folded || 'attribute' END ) AND (source_file.lang <> 'dependency_lock' OR s.file_id = r.file_id) AND {CSharpTypeReferenceCandidatePredicateSql} + AND source_file.lang <> 'markdown' AND r.target_qualifier IS NOT NULL AND r.target_qualifier NOT LIKE char(31) || 'receiver:%' AND ( @@ -533,6 +566,7 @@ THEN r.symbol_name_folded || 'attribute' END OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND {CSharpTypeReferenceCandidatePredicateSql} + AND source_file.lang <> 'markdown' AND r.target_qualifier IS NULL AND s.file_id = r.file_id AND source.container_name IS NOT NULL @@ -564,6 +598,7 @@ THEN r.symbol_name_folded || 'attribute' END OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND {CSharpTypeReferenceCandidatePredicateSql} + AND source_file.lang <> 'markdown' AND r.target_qualifier IS NULL AND (source_file.lang <> 'dependency_lock' OR s.file_id = r.file_id) AND source.container_qualified_name IS NOT NULL @@ -591,6 +626,7 @@ THEN r.symbol_name_folded || 'attribute' END OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND {CSharpTypeReferenceCandidatePredicateSql} + AND source_file.lang <> 'markdown' AND r.target_qualifier IS NULL AND s.file_id = r.file_id AND NOT EXISTS ( @@ -616,6 +652,7 @@ THEN r.symbol_name_folded || 'attribute' END OR (source_file.lang = 'ambiguous_m' AND target_file.lang IN ('matlab', 'objc')) ) AND {CSharpTypeReferenceCandidatePredicateSql} + AND source_file.lang <> 'markdown' AND r.target_qualifier IS NULL AND (source_file.lang <> 'dependency_lock' OR s.file_id = r.file_id) AND source.container_name IS NOT NULL @@ -703,7 +740,7 @@ AND target_file.lang IN ('matlab', 'objc') AND target_file.path || char(31) || COALESCE(target.container_qualified_name, target.container_name, '') || char(31) || COALESCE(target.name, '') = unique_family.family_key - WHERE source_file.lang <> 'csharp' + WHERE source_file.lang NOT IN ('csharp', 'markdown') AND ( r.target_qualifier IS NULL OR source_file.lang IN ( diff --git a/src/CodeIndex/Indexer/MarkdownAnchorIdentity.cs b/src/CodeIndex/Indexer/MarkdownAnchorIdentity.cs new file mode 100644 index 000000000..286873caa --- /dev/null +++ b/src/CodeIndex/Indexer/MarkdownAnchorIdentity.cs @@ -0,0 +1,335 @@ +using System.Net; +using System.Text; + +namespace CodeIndex.Indexer; + +internal static class MarkdownAnchorIdentity +{ + internal static string NormalizeHeadingFragment(string value) => + Slugify(DecodeFragment(value)); + + internal static string NormalizeExplicitAnchorDefinition(string value) => + WebUtility.HtmlDecode(value.Trim()); + + internal static string DecodeExplicitAnchorFragment(string value) => + DecodeFragment(value); + + private static string DecodeFragment(string value) + { + var anchor = value.Trim(); + if (anchor.Length >= 2 && anchor[0] == '<' && anchor[^1] == '>') + anchor = anchor[1..^1].Trim(); + if (anchor.Length == 0) + return string.Empty; + + anchor = WebUtility.HtmlDecode(anchor); + try + { + anchor = Uri.UnescapeDataString(anchor); + } + catch (UriFormatException) + { + // Preserve malformed fragments as unresolved evidence instead of dropping them. + // 不正な fragment も削除せず、未解決の参照根拠として保持する。 + } + + return anchor; + } + + private static string Slugify(string value) + { + var normalized = value.Normalize(NormalizationForm.FormKC); + var builder = new StringBuilder(normalized.Length); + var previousDash = false; + foreach (var originalRune in normalized.EnumerateRunes()) + { + var rune = Rune.ToLowerInvariant(originalRune); + if (Rune.IsLetterOrDigit(rune) || rune.Value is '_' or '-') + { + builder.Append(rune.ToString()); + previousDash = rune.Value == '-'; + } + else if (Rune.IsWhiteSpace(rune) && builder.Length > 0 && !previousDash) + { + builder.Append('-'); + previousDash = true; + } + } + + while (builder.Length > 0 && builder[^1] == '-') + builder.Length--; + + return builder.ToString(); + } + + internal static string CreateUniqueHeadingIdentity(string headingText, HashSet usedIdentities) + { + var baseIdentity = Slugify(FlattenHeadingInlineMarkup(headingText)); + if (baseIdentity.Length == 0) + return string.Empty; + if (usedIdentities.Add(baseIdentity)) + return baseIdentity; + + for (var suffix = 1; ; suffix++) + { + var candidate = $"{baseIdentity}-{suffix}"; + if (usedIdentities.Add(candidate)) + return candidate; + } + } + + private static string FlattenHeadingInlineMarkup(string value) + { + var builder = new StringBuilder(value.Length); + AppendHeadingInlineText(value, builder); + return WebUtility.HtmlDecode(builder.ToString()); + } + + private static void AppendHeadingInlineText(string value, StringBuilder builder) + { + for (var index = 0; index < value.Length;) + { + if (value[index] == '\\' && index + 1 < value.Length) + { + builder.Append(value[index + 1]); + index += 2; + continue; + } + + if (value[index] == '<' && TryFindHtmlTagEnd(value, index, out var tagEnd)) + { + index = tagEnd + 1; + continue; + } + + if (value[index] == '`') + { + var delimiterLength = 1; + while (index + delimiterLength < value.Length + && value[index + delimiterLength] == '`') + { + delimiterLength++; + } + + var closing = value.IndexOf( + new string('`', delimiterLength), + index + delimiterLength, + StringComparison.Ordinal); + if (closing >= 0) + { + builder.Append(value, index + delimiterLength, closing - index - delimiterLength); + index = closing + delimiterLength; + continue; + } + } + + if (value[index] == '_' + && TryFindClosingEmphasisDelimiter(value, index, out var emphasisEnd, out var emphasisLength)) + { + AppendHeadingInlineText( + value[(index + emphasisLength)..emphasisEnd], + builder); + index = emphasisEnd + emphasisLength; + continue; + } + + var labelStart = value[index] == '!' + && index + 1 < value.Length + && value[index + 1] == '[' + ? index + 1 + : index; + if (value[labelStart] == '[' + && TryFindClosingDelimiter(value, labelStart, '[', ']', out var labelEnd)) + { + AppendHeadingInlineText(value[(labelStart + 1)..labelEnd], builder); + index = labelEnd + 1; + if (index < value.Length + && value[index] == '(' + && TryFindClosingDelimiter(value, index, '(', ')', out var destinationEnd)) + { + index = destinationEnd + 1; + } + else if (index < value.Length + && value[index] == '[' + && TryFindClosingDelimiter(value, index, '[', ']', out var referenceEnd)) + { + index = referenceEnd + 1; + } + continue; + } + + builder.Append(value[index]); + index++; + } + } + + private static bool TryFindClosingEmphasisDelimiter( + string value, + int openingIndex, + out int closingIndex, + out int delimiterLength) + { + delimiterLength = 1; + while (openingIndex + delimiterLength < value.Length + && value[openingIndex + delimiterLength] == '_') + { + delimiterLength++; + } + + closingIndex = -1; + var contentStart = openingIndex + delimiterLength; + if (contentStart >= value.Length + || char.IsWhiteSpace(value[contentStart]) + || IsIntrawordUnderscore(value, openingIndex)) + { + return false; + } + + for (var index = contentStart; index < value.Length;) + { + if (value[index] != '_') + { + index++; + continue; + } + + var runLength = 1; + while (index + runLength < value.Length + && value[index + runLength] == '_') + { + runLength++; + } + + if (runLength == delimiterLength + && index > contentStart + && !char.IsWhiteSpace(value[index - 1]) + && !IsIntrawordUnderscore(value, index)) + { + closingIndex = index; + return true; + } + + index += runLength; + } + + return false; + } + + private static bool IsIntrawordUnderscore(string value, int index) => + index > 0 + && index + 1 < value.Length + && char.IsLetterOrDigit(value[index - 1]) + && char.IsLetterOrDigit(value[index + 1]); + + private static bool TryFindClosingDelimiter( + string value, + int openingIndex, + char opening, + char closing, + out int closingIndex) + { + var depth = 1; + for (var index = openingIndex + 1; index < value.Length; index++) + { + if (value[index] == '\\') + { + index++; + continue; + } + if (value[index] == opening) + { + depth++; + continue; + } + if (value[index] != closing || --depth != 0) + continue; + + closingIndex = index; + return true; + } + + closingIndex = -1; + return false; + } + + private static bool TryFindHtmlTagEnd(string value, int openingIndex, out int tagEnd) + { + tagEnd = -1; + if (IsMarkdownAutolink(value, openingIndex)) + return false; + + var index = openingIndex + 1; + if (index >= value.Length) + return false; + + if (value.AsSpan(index).StartsWith("!--", StringComparison.Ordinal)) + { + var commentEnd = value.IndexOf("-->", index + 3, StringComparison.Ordinal); + if (commentEnd < 0) + return false; + tagEnd = commentEnd + 2; + return true; + } + + if (value[index] is '!' or '?') + index++; + else + { + if (value[index] == '/') + index++; + var nameStart = index; + while (index < value.Length + && (char.IsAsciiLetterOrDigit(value[index]) || value[index] is '-' or ':')) + { + index++; + } + if (index == nameStart + || (index < value.Length + && !char.IsWhiteSpace(value[index]) + && value[index] is not ('/' or '>'))) + { + return false; + } + } + + var quote = '\0'; + for (; index < value.Length; index++) + { + var current = value[index]; + if (quote != '\0') + { + if (current == quote) + quote = '\0'; + continue; + } + if (current is '"' or '\'') + { + quote = current; + continue; + } + if (current == '>') + { + tagEnd = index; + return true; + } + } + + return false; + } + + private static bool IsMarkdownAutolink(string value, int openingIndex) + { + var closing = value.IndexOf('>', openingIndex + 1); + if (closing < 0) + return false; + + var target = value.AsSpan(openingIndex + 1, closing - openingIndex - 1); + if (target.IsEmpty || target.IndexOfAny(" \t\r\n") >= 0) + return false; + + return target.StartsWith("http://", StringComparison.OrdinalIgnoreCase) + || target.StartsWith("https://", StringComparison.OrdinalIgnoreCase) + || target.StartsWith("mailto:", StringComparison.OrdinalIgnoreCase) + || target.IndexOf('@') > 0; + } +} diff --git a/src/CodeIndex/Indexer/References/Languages/MarkupSchemaReferenceExtractor.cs b/src/CodeIndex/Indexer/References/Languages/MarkupSchemaReferenceExtractor.cs index d61456936..666e876d7 100644 --- a/src/CodeIndex/Indexer/References/Languages/MarkupSchemaReferenceExtractor.cs +++ b/src/CodeIndex/Indexer/References/Languages/MarkupSchemaReferenceExtractor.cs @@ -11,6 +11,7 @@ internal sealed class MarkupState public bool InMarkdownFence { get; set; } public char MarkdownFenceChar { get; set; } public int MarkdownFenceLength { get; set; } + public IReadOnlyDictionary? MarkdownReferenceTargets { get; set; } public bool InHtmlComment { get; set; } public string? HtmlRawTextTag { get; set; } public int GraphQLBraceDepth { get; set; } @@ -110,9 +111,17 @@ internal sealed class MarkupState RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex MarkdownReferenceLinkRegex = new( - @"!?\[[^\]\r\n]+\]\[(?