From d8f7a899e163c4b2d195e00fba401bbd306e5c6f Mon Sep 17 00:00:00 2001 From: Widthdom Date: Mon, 27 Jul 2026 23:05:26 +0900 Subject: [PATCH 1/3] Report search selector population accounting (#4843) --- DEVELOPER_GUIDE.md | 8 +- USER_GUIDE.md | 33 ++- changelog.d/unreleased/4843.fixed.md | 25 ++ src/CodeIndex/Cli/JsonOutputContracts.cs | 16 ++ .../Cli/QueryCommandRunner.Ndjson.cs | 142 ++++++---- .../Cli/QueryCommandRunner.ResultEnvelopes.cs | 27 ++ .../Cli/QueryCommandRunner.Search.cs | 20 +- .../Cli/QueryCommandRunner.SearchConstants.cs | 2 + .../Cli/QueryCommandRunner.SearchRecipes.cs | 114 +++++++- .../Cli/QueryCommandRunner.SearchResults.cs | 174 ++++++++++--- src/CodeIndex/Cli/SearchAuditRecipes.cs | 63 ++++- .../QueryCommandRunnerSearchTests.cs | 243 ++++++++++++++++++ 12 files changed, 748 insertions(+), 119 deletions(-) create mode 100644 changelog.d/unreleased/4843.fixed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 2a25324f71..eff4724c81 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1551,10 +1551,10 @@ access. | Bounded high-volume responses | `search`, `definition`, `find`, `status`, `hotspots`, `references`, `callers`, `callees`, `symbols`, `files`, `languages`, `impact`, and `map` accept shared bounded-response controls where their schema exposes them. Newly emitted opaque `--cursor ` values bind the offset to the command/query/filter selection and index generation; legacy `response:v1::` cursors remain accepted for transition. Reuse with changed selection or generation fails with restart-required guidance. `search --format compact`, `symbols --format compact`, and `files --format compact` auto-select the bounded contract, while `search --json=array --json-envelope` provides the opt-in array envelope and `languages --json` selects it when paging or `--max-json-bytes` is requested. Existing compact roots and location rows remain compatible while adding shared metadata. Metadata reports `returned_count`, authoritative `total_count` where available, `omitted_count`, `remaining_count`, `cursor_offset`, `page_limit`, `has_more`, `next_cursor`, `result_stable_at`, `pagination_window_limit`, and `pagination_window_exhausted`. The safety window is 10,000 rows; exhaustion suppresses `next_cursor` rather than returning a cursor that the next request would reject. Pageable commands pass the cursor offset into their database/scan layer instead of serializing an `offset + limit` prefix. `find --all` partial scans encode the next path/line in the opaque cursor so replay continues after the last scanned line. `hotspots` and `impact` page their active primary nested collection as `results`, identify it with `metadata.primary_collection`, and retain scalar/container evidence in `metadata.response_context`; dotted fields such as `callers.path,callers.depth` select that collection and project its rows. The final newline is included in `--max-json-bytes`, and trailing whole rows are removed until the complete envelope fits. `definition` remains metadata-only by default; explicit `--body` content is retained for `body`, `body_content`, or `all`, and suppressed when the projection excludes it. `map --sections` remains its section-level projection, while dotted bounded fields page a selected array section with section-specific totals and scalar projections skip unused ranked arrays. | | Bounded response edge cases | `impact` applies the cursor offset only to the selected nested collection so definition pages do not repeat or alter caller/fallback mode. Plain `map --compact` preserves its established section arrays and truncation payload; a collection projection is rejected when `--summary-only` or an excluding `--sections` filter would remove it. Explicit definition body fields override compact defaults. Profile and verbose records are moved into `metadata.stream_control_records`, and parser/capture failures emit an error envelope only when it fits the active hard byte cap. | | `--count --json` envelope | Count-only JSON for `search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `impact`, and `unused` is a single automation-oriented object. It always includes `count`, applied `query_context`, freshness metadata (`indexed_file_count`, `indexed_at`, `freshness_available`), and trust flags `degraded` / `authoritative_count`; commands with matched-file totals also include `files` and the older `file_count` compatibility alias. `file_count` carries the same value as `files`, remains for compatibility, and is not scheduled for removal before the next major release. `unused --count --json` also includes `returned_bucket_counts`, `returned_contract_domain_counts`, and `summary.by_bucket` / `summary.by_confidence` / `summary.by_contract_domain`. `authoritative_count=false` means a readiness or graph/exact trust signal made the count non-authoritative, while the freshness fields describe the indexed snapshot used for the count. | -| Recipe row selection | Row-producing recipe paths share `ApplySearchOutputSelection`: `--first-per-file` and deterministic `--sample` run before the effective per-query / remaining total limit. Sample fetch envelopes are sized from at least the requested sample target. Aggregate and compact query DTOs expose nullable `selection_reason` / `selection_omitted_count`; issue-draft source DTOs preserve them and recipe NDJSON forwards their aggregate to the terminal record. Selection-only omission updates matched/omitted lower bounds without setting `truncated`, `has_more`, or `next_cursor`. When a later limit truncates selected rows, limit truncation remains visible but `next_cursor` is suppressed because raw database cursors cannot preserve selector state; incoming `--cursor` values are rejected with either selector for the same reason. Generated compact and issue-draft replay commands retain the selector. Count, aggregation, and summary-only compact shapes reject `--first-per-file` / `--sample`, while every recipe shape rejects grouped-only `--per-file-limit`. | +| Recipe row selection | Row-producing recipe paths share `ApplySearchOutputSelection`: `--first-per-file` and fixed-seed deterministic `--sample` run before the effective per-query / remaining total limit. Sample fetch envelopes are sized from at least the requested sample target. Aggregate/compact query DTOs, run summaries, issue-draft source DTOs, and NDJSON terminals expose `source_total`, `selected_total`, `returned`, `selector_omitted_count`, and `limit_omitted_count`; `source_total_authoritative` / `source_total_lower_bound` distinguish complete populations from bounded observations. Their ordered `selectors` entries preserve each stage's input/output/omission counts plus sample size, mode, and seed, while nullable `selection_reason` / `selection_omitted_count` remain compatibility summaries. Search `query_context.row_selectors` records the applied selector configuration. Selection-only omission updates matched/omitted lower bounds without setting `truncated`, `has_more`, or `next_cursor`. When a later limit truncates selected rows, limit truncation remains visible but `next_cursor` is suppressed because raw database cursors cannot preserve selector state; incoming `--cursor` values are rejected with either selector for the same reason. Generated compact and issue-draft replay commands retain the selector. Count, aggregation, and summary-only compact shapes reject `--first-per-file` / `--sample`, while every recipe shape rejects grouped-only `--per-file-limit`. | | Recipe SARIF | `search --recipe --format sarif` emits one result per bounded recipe result. Rule IDs use `recipe/query`; standard `fingerprints.cdidx/v1` values are derived from the normalized source location; result properties preserve recipe/query identity, severity, confidence, and per-query truncation; run properties preserve scope, applied result limits, aggregate counts, and conservative omitted-result metadata. Bound SARIF with `--limit` / `--total-limit`; row selectors such as `--sample`, `--first-per-file`, and `--per-file-limit` are rejected instead of being silently ignored. Recipe severity maps `critical` / `high` to `error`, `medium` to `warning`, and `low` / `info` to `note`. | | Recipe classifier output | Recipe run JSON may add `audit_classifications` to individual `CompactSearchResult` rows when a recipe classifier can classify the hit, and query/count payloads may add `classifier_counts` when classified rows are present. These fields are additive; use them to separate triage domains such as DTO/result-wrapper `.Result` properties versus Task/ValueTask blocking waits without changing the raw search query. | -| NDJSON terminal records | Default NDJSON for `search`, `symbols`, and `files` appends one final `terminal_record` after result rows; search also emits it for zero-result responses, while raw `symbols` and `files` keep zero-result NDJSON empty. Recipe/audit search row streams share the same writer. Terminals report returned and observed total counts, `total_count_authoritative` / `total_count_lower_bound`, selection or interruption reason, applied limits, omitted rows, and recovery guidance. `--max-json-bytes` covers the complete stdout stream, including newlines and this terminal record; a cap that cannot fit the terminal fails before stdout. Capped output rejects `--profile`, `--verbose`, and `--json-envelope`. Byte-cap partial output exits with `CommandExitCodes.PartialResult` (`11`) unless `--allow-partial` explicitly opts into exit `0`. `--results-only` is the explicit terminal-record opt-out for these NDJSON row streams and is rejected with array, compact, summary, or count output. | +| NDJSON terminal records | Default NDJSON for `search`, `symbols`, and `files` appends one final `terminal_record` after result rows; search also emits it for zero-result responses, while raw `symbols` and `files` keep zero-result NDJSON empty. Recipe/audit search row streams share the same writer. Terminals report returned and observed total counts, `total_count_authoritative` / `total_count_lower_bound`, selection or interruption reason, applied limits, omitted rows, and recovery guidance. `--max-json-bytes` covers the complete stdout stream, including newlines and this terminal record; when additive selector-accounting fields prevent the terminal from fitting, the writer omits those optional fields before declaring the terminal impossible. A cap that still cannot fit the terminal fails before stdout. Capped output rejects `--profile`, `--verbose`, and `--json-envelope`. Byte-cap partial output exits with `CommandExitCodes.PartialResult` (`11`) unless `--allow-partial` explicitly opts into exit `0`. `--results-only` is the explicit terminal-record opt-out for these NDJSON row streams and is rejected with array, compact, summary, or count output. | | `outline` / `unused` cursor binding | `outline --json` accepts `--kind `, `--limit` / `--top`, opaque `--cursor `, and `--outline-fields ` for bounded machine output. Controlled outline responses keep the normal envelope and add `total_symbol_count`, `returned_symbol_count`, `cursor_offset`, `next_cursor`, `has_more`, and `result_stable_at`, plus `kind_filter` and `selected_fields` when active. `outline` and `unused` cursors bind their offset to the normalized path/scope, filters, ordering, and index generation; reuse after changing those inputs or refreshing the index fails with explicit restart-required guidance. Legacy `outline:` / `unused:` inputs remain accepted for transition, but every newly emitted cursor is opaque and bound. | | `hotspots --json` grouping semantics | `hotspots` and MCP `symbol_hotspots` emit `grouped_by`, `grouping_unit`, `count_kind`, `limit_applies_to`, `score_fields`, `ranking_fields`, and matching `query_context` fields. `--limit` applies to returned symbols, files, name/kind groups, or SQL statements; `--count` ignores `--limit` and reports total groups. Explicit `statement` grouping is SQL-only (`--lang sql` / `lang: "sql"`). | | `--json-envelope` commands | Applies to `search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `excerpt`, `map`, `inspect`, `outline`, `status`, `validate`, `languages`, `impact`, `deps`, `unused`, and `hotspots`. | @@ -4735,10 +4735,10 @@ help はすべてこのレジストリを参照します。field 名は大文字 | 高ボリューム応答の bounded 契約 | `search`、`definition`、`find`、`status`、`hotspots`、`references`、`callers`、`callees`、`symbols`、`files`、`languages`、`impact`、`map` は、それぞれの schema が公開する共通 bounded-response control に対応します。新しく発行する opaque な `--cursor ` は offset を command / query / filter と index generation に束縛し、移行用に legacy の `response:v1::` も受理します。選択条件または generation を変えて再利用すると restart-required の案内付きで失敗します。`search --format compact`、`symbols --format compact`、`files --format compact` は bounded 契約を自動選択し、`search --json=array --json-envelope` は opt-in の array envelope、`languages --json` は paging または `--max-json-bytes` 指定時に同じ契約を使います。既存 compact の root と location row は維持したまま共通 metadata を追加します。metadata は `returned_count`、取得可能な場合は authoritative な `total_count`、`omitted_count`、`remaining_count`、`cursor_offset`、`page_limit`、`has_more`、`next_cursor`、`result_stable_at`、`pagination_window_limit`、`pagination_window_exhausted` を返します。safety window は 10,000 row で、上限到達時は次の request が拒否する cursor を返さず `next_cursor` を抑止します。pageable command は `offset + limit` 件を serialize せず、cursor offset を database / scan layer へ渡します。`find --all` の partial scan cursor は次の path / line を保持し、再利用時は最後に scan した line の次から継続します。`hotspots` と `impact` は active な主要 nested collection を `results` としてページングし、`metadata.primary_collection` でその名前を示し、scalar / container evidence は `metadata.response_context` に保持します。`callers.path,callers.depth` のような dotted field で collection と row field を同時に選べます。`--max-json-bytes` は最後の改行を含み、完全な envelope が収まるまで末尾の完全な row を省略します。`definition` は既定で metadata-only のままで、明示的な `--body` は `body`、`body_content`、`all` で保持し、それ以外の projection では materialize 前に抑止します。`map --sections` は section-level projection として残り、dotted な bounded field は選択した array section を section 固有の総件数付きでページングし、scalar projection は不要な ranking array を構築しません。 | | bounded 応答の edge case | `impact` は選択された nested collection だけに cursor offset を適用するため、definition page の重複や caller / fallback mode の変化を防ぎます。通常の `map --compact` は既存の section array と truncation payload を維持し、collection projection が `--summary-only` または除外する `--sections` filter で失われる組み合わせは拒否します。明示的な definition body field は compact default より優先します。profile / verbose record は `metadata.stream_control_records` へ移し、parser / capture failure の error envelope は active な hard byte cap に収まる場合だけ出力します。 | | `--count --json` envelope | `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`find`、`impact`、`unused` の count-only JSON は単一の自動化向け object です。常に `count`、適用済み `query_context`、freshness metadata（`indexed_file_count`、`indexed_at`、`freshness_available`）、trust flag の `degraded` / `authoritative_count` を含みます。matched-file total を持つ command は `files` と古い互換 alias の `file_count` も含みます。`file_count` は `files` と同じ値を持つ互換用 field として残り、少なくとも次の major release までは削除予定はありません。`unused --count --json` は `returned_bucket_counts`、`returned_contract_domain_counts`、`summary.by_bucket` / `summary.by_confidence` / `summary.by_contract_domain` も含みます。`authoritative_count=false` は readiness または graph/exact trust signal により count が authoritative ではないことを示し、freshness field は count に使った index snapshot を説明します。 | -| recipe row selection | row を返す recipe path は `ApplySearchOutputSelection` を共有し、`--first-per-file` と決定的な `--sample` を、有効な query ごとの limit / 残り total limit より先に適用します。sample 用 fetch envelope は少なくとも要求 sample 数を基準に sizing します。aggregate / compact の query DTO は nullable な `selection_reason` / `selection_omitted_count` を公開し、issue-draft の source DTO もこれらを保持し、recipe NDJSON は集計値を terminal record へ渡します。selection だけによる省略は matched / omitted の lower bound を更新しますが、`truncated`、`has_more`、`next_cursor` は設定しません。後続の limit が選択済み row を truncate する場合、limit truncation は表示しますが raw database cursor は selector state を保持できないため `next_cursor` を抑止し、同じ理由で selector と受け取った `--cursor` の併用も拒否します。compact / issue-draft の生成 replay command は selector を保持します。count、aggregation、summary-only compact の shape は `--first-per-file` / `--sample` を拒否し、すべての recipe shape は grouped 専用の `--per-file-limit` を拒否します。 | +| recipe row selection | row を返す recipe path は `ApplySearchOutputSelection` を共有し、`--first-per-file` と固定 seed の決定的な `--sample` を、有効な query ごとの limit / 残り total limit より先に適用します。sample 用 fetch envelope は少なくとも要求 sample 数を基準に sizing します。aggregate / compact の query DTO、run summary、issue-draft の source DTO、NDJSON terminal は `source_total`、`selected_total`、`returned`、`selector_omitted_count`、`limit_omitted_count` を公開し、`source_total_authoritative` / `source_total_lower_bound` で完全な population と bounded な観測を区別します。適用順の `selectors` entry は各段階の input / output / omission count と sample の size / mode / seed を保持し、nullable な `selection_reason` / `selection_omitted_count` は互換用 summary として維持します。search の `query_context.row_selectors` は適用済み selector 設定を記録します。selection だけによる省略は matched / omitted の lower bound を更新しますが、`truncated`、`has_more`、`next_cursor` は設定しません。後続の limit が選択済み row を truncate する場合、limit truncation は表示しますが raw database cursor は selector state を保持できないため `next_cursor` を抑止し、同じ理由で selector と受け取った `--cursor` の併用も拒否します。compact / issue-draft の生成 replay command は selector を保持します。count、aggregation、summary-only compact の shape は `--first-per-file` / `--sample` を拒否し、すべての recipe shape は grouped 専用の `--per-file-limit` を拒否します。 | | Recipe SARIF | `search --recipe --format sarif` は、上限付き recipe result ごとに result を1件出力します。rule ID は `recipe/query` を使い、標準の `fingerprints.cdidx/v1` は正規化済み source location から導出します。result properties は recipe/query identity、severity、confidence、query ごとの truncation を保持し、run properties は scope、適用済み result limit、集計 count、保守的な omitted-result metadata を保持します。SARIF の上限には `--limit` / `--total-limit` を使い、`--sample`、`--first-per-file`、`--per-file-limit` のような row selector は黙って無視せず拒否します。recipe severity は `critical` / `high` を `error`、`medium` を `warning`、`low` / `info` を `note` に対応付けます。 | | Recipe classifier output | recipe classifier が hit を分類できる場合、recipe run JSON は個別の `CompactSearchResult` row に `audit_classifications` を追加することがあり、分類済み row がある query / count payload は `classifier_counts` を追加することがあります。これらは additive field です。raw search query を変えずに、DTO / result-wrapper の `.Result` property と Task / ValueTask の blocking wait などの triage domain を分離するために使います。 | -| NDJSON terminal record | `search`、`symbols`、`files` の既定 NDJSON は result row の後に最後の `terminal_record` を 1 件追加します。`search` は 0 件応答にも終端を出力しますが、raw `symbols` / `files` の 0 件 NDJSON は空のままです。recipe / audit search の row stream も同じ writer を使います。終端は返却件数と観測済み総件数、`total_count_authoritative` / `total_count_lower_bound`、selection または中断理由、適用上限、省略行数、復旧案内を報告します。`--max-json-bytes` は改行と終端レコードを含む stdout stream 全体を対象にし、終端自体が収まらない cap は stdout 出力前に失敗します。上限付き出力は `--profile`、`--verbose`、`--json-envelope` を拒否します。byte cap による部分出力は、`--allow-partial` で終了コード `0` を明示許可しない限り `CommandExitCodes.PartialResult`（`11`）を返します。`--results-only` はこれらの NDJSON row stream から終端レコードを明示的に除外するための option であり、array / compact / summary / count 出力との組み合わせは拒否されます。 | +| NDJSON terminal record | `search`、`symbols`、`files` の既定 NDJSON は result row の後に最後の `terminal_record` を 1 件追加します。`search` は 0 件応答にも終端を出力しますが、raw `symbols` / `files` の 0 件 NDJSON は空のままです。recipe / audit search の row stream も同じ writer を使います。終端は返却件数と観測済み総件数、`total_count_authoritative` / `total_count_lower_bound`、selection または中断理由、適用上限、省略行数、復旧案内を報告します。`--max-json-bytes` は改行と終端レコードを含む stdout stream 全体を対象にし、追加 selector-accounting field が原因で終端が収まらない場合は、終端自体を不可能と判定する前にそれらの任意 field を省略します。それでも終端が収まらない cap は stdout 出力前に失敗します。上限付き出力は `--profile`、`--verbose`、`--json-envelope` を拒否します。byte cap による部分出力は、`--allow-partial` で終了コード `0` を明示許可しない限り `CommandExitCodes.PartialResult`（`11`）を返します。`--results-only` はこれらの NDJSON row stream から終端レコードを明示的に除外するための option であり、array / compact / summary / count 出力との組み合わせは拒否されます。 | | `outline` / `unused` cursor の束縛 | `outline --json` は bounded な機械向け出力として `--kind `、`--limit` / `--top`、opaque な `--cursor `、`--outline-fields ` を受け付けます。制御付き outline 応答は通常の envelope を維持し、`total_symbol_count`、`returned_symbol_count`、`cursor_offset`、`next_cursor`、`has_more`、`result_stable_at` を追加し、該当時は `kind_filter` と `selected_fields` も返します。`outline` と `unused` の cursor は offset を正規化済み path/scope、filter、ordering、index generation に束縛するため、条件変更後または index 更新後の再利用は restart-required の明示案内付きで失敗します。移行用に legacy の `outline:` / `unused:` 入力は受理しますが、新しく出力する cursor はすべて opaque かつ束縛済みです。 | | `hotspots --json` grouping semantics | `hotspots` と MCP `symbol_hotspots` は `grouped_by`、`grouping_unit`、`count_kind`、`limit_applies_to`、`score_fields`、`ranking_fields` と、対応する `query_context` field を返します。`--limit` は返却される symbol、file、name/kind group、SQL statement に適用されます。`--count` は `--limit` を無視し、total group 数を返します。明示的な `statement` grouping は SQL 専用です（`--lang sql` / `lang: "sql"`）。 | | `--json-envelope` 対象 command | `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`find`、`excerpt`、`map`、`inspect`、`outline`、`status`、`validate`、`languages`、`impact`、`deps`、`unused`、`hotspots`。 | diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 22f56ac237..d7168739d3 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1383,11 +1383,22 @@ aggregations support `--count-by path|file|symbol|origin|return-type|subsystem`, `--group-by file|symbol|origin|return-type|subsystem --count`, and `--unique path|file|symbol|origin|return-type|subsystem`. Row-producing recipe modes (text, aggregate JSON, compact JSON, NDJSON, and -issue drafts) apply `--first-per-file` and deterministic `--sample ` before +issue drafts) apply `--first-per-file` and fixed-seed deterministic +`--sample ` before the effective per-query `--limit` / cross-query `--total-limit`. Aggregate JSON -and compact query objects report `selection_reason` and -`selection_omitted_count` when selection removes rows; issue-draft `source` -objects and NDJSON terminal records report the same fields. Selection-only +and compact query objects, run summaries, issue-draft `source` objects, and +NDJSON terminal records distinguish `source_total`, `selected_total`, +`returned`, `selector_omitted_count`, and `limit_omitted_count`. Their +`selectors` array records each applied selector in execution order, including +per-stage input/output/omission counts and the sample size, mode, and seed. +`source_total_authoritative` says whether the bounded fetch observed the whole +source population; otherwise `source_total_lower_bound` is also present. The +older `selection_reason` and `selection_omitted_count` fields remain as +compatibility summaries. Search `query_context.row_selectors` exposes every +applied selector with the same sample mode and seed. When a hard +`--max-json-bytes` cap cannot fit the additive accounting fields in an NDJSON +terminal, the writer omits those optional fields before failing the terminal +budget; the compatibility selection fields remain. Selection-only omission contributes to matched and omitted counts but does not set `truncated`, `has_more`, or `next_cursor`. If a later limit also omits selected rows, `truncated` / `has_more` are set but `next_cursor` is suppressed because a raw @@ -4506,10 +4517,18 @@ recipe count output は `--format count --summary-only --max-json-bytes ` に aggregate count、query ごとの count、query freshness だけを出力できます。recipe の count aggregation は `--count-by path|file|symbol|origin|return-type|subsystem`、 `--group-by file|symbol|origin|return-type|subsystem --count`、`--unique path|file|symbol|origin|return-type|subsystem` に対応します。 row を返す recipe mode（text、aggregate JSON、compact JSON、NDJSON、issue draft）は、 -`--first-per-file` と決定的な `--sample ` を、有効な query ごとの `--limit` / +`--first-per-file` と固定 seed の決定的な `--sample ` を、有効な query ごとの `--limit` / query 全体の `--total-limit` より先に適用します。aggregate JSON / compact の query object は -selection で row が省略された場合に `selection_reason` と `selection_omitted_count` を返し、 -issue-draft の `source` object と NDJSON terminal record も同じ field を返します。 +run summary、issue-draft の `source` object、NDJSON terminal record と同様に、 +`source_total`、`selected_total`、`returned`、`selector_omitted_count`、 +`limit_omitted_count` を分けて返します。`selectors` array は適用順の各 selector について、 +各段階の入力件数、出力件数、省略件数、および sample の size / mode / seed を記録します。 +bounded fetch が source population 全体を観測できたかは `source_total_authoritative` で示し、 +そうでない場合は `source_total_lower_bound` も返します。従来の `selection_reason` と +`selection_omitted_count` は互換用 summary として維持します。 +search の `query_context.row_selectors` も適用済み selector と同じ sample mode / seed を公開します。 +hard な `--max-json-bytes` cap で NDJSON terminal に追加 accounting field が収まらない場合、 +terminal budget 自体を失敗させる前にこれらの任意 field を省略し、互換用 selection field は維持します。 selection だけによる省略は matched / omitted count に含まれますが、`truncated`、 `has_more`、`next_cursor` は設定しません。後続の limit でも選択済み row が省略される場合は `truncated` / `has_more` を設定しますが、raw cursor では row-selection state を保持できないため diff --git a/changelog.d/unreleased/4843.fixed.md b/changelog.d/unreleased/4843.fixed.md new file mode 100644 index 0000000000..eb25fcdf08 --- /dev/null +++ b/changelog.d/unreleased/4843.fixed.md @@ -0,0 +1,25 @@ +--- +category: fixed +issues: + - 4843 +affected: + - src/CodeIndex/Cli/JsonOutputContracts.cs + - src/CodeIndex/Cli/QueryCommandRunner.Ndjson.cs + - src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs + - src/CodeIndex/Cli/QueryCommandRunner.Search.cs + - src/CodeIndex/Cli/QueryCommandRunner.SearchConstants.cs + - src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs + - src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs + - src/CodeIndex/Cli/SearchAuditRecipes.cs + - tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs + - USER_GUIDE.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Sampling and first-per-file output now reports the source, selected, and returned populations separately (#4843)** — recipe JSON, compact output, issue drafts, and NDJSON terminals now expose per-selector accounting, limit omissions, deterministic sample mode/seed, and bounded-source authority. Search query context lists every applied selector, while count and other non-row recipe modes reject selectors with a stable usage error. + +## 日本語 + +- **sample / first-per-file 出力で選択前・選択後・返却後の件数を分けて報告するようになりました (#4843)** — recipe JSON、compact 出力、issue draft、NDJSON terminal は selector ごとの件数、limit による省略、決定的 sample の mode / seed、bounded な source 件数の authority を公開します。search の query context は適用済み selector をすべて列挙し、count など row を返さない recipe mode は selector を安定した usage error で拒否します。 diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index bf9f0a6850..70ab395512 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -593,6 +593,22 @@ internal sealed record JsonStreamDoneResult( [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? SelectionReason = null, [property: JsonPropertyName("selection_omitted_count")] [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] int? SelectionOmittedCount = null, + [property: JsonPropertyName("source_total")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] int? SourceTotal = null, + [property: JsonPropertyName("source_total_authoritative")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] bool? SourceTotalAuthoritative = null, + [property: JsonPropertyName("source_total_lower_bound")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] int? SourceTotalLowerBound = null, + [property: JsonPropertyName("selected_total")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] int? SelectedTotal = null, + [property: JsonPropertyName("returned")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] int? Returned = null, + [property: JsonPropertyName("selector_omitted_count")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] int? SelectorOmittedCount = null, + [property: JsonPropertyName("limit_omitted_count")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] int? LimitOmittedCount = null, + [property: JsonPropertyName("selectors")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] List? Selectors = null, [property: JsonPropertyName("interruption_reason")] [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? InterruptionReason = null, [property: JsonPropertyName("truncation_reason")] diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Ndjson.cs b/src/CodeIndex/Cli/QueryCommandRunner.Ndjson.cs index 9277c44a47..8f54dcaac9 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Ndjson.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Ndjson.cs @@ -41,13 +41,53 @@ private static NdjsonStreamWriteResult WriteNdjsonStream( bool totalCountAuthoritative = true, string? truncationReason = null, string? selectionReason = null, - int? selectionOmittedCount = null) + int? selectionOmittedCount = null, + int? sourceTotal = null, + bool? sourceTotalAuthoritative = null, + int? selectedTotal = null, + int? selectorOmittedCount = null, + int? limitOmittedCount = null, + List? selectors = null) { if (options.ResultsOnly) return WriteResultOnlyNdjson(records, options); var emittedRecords = records.Count; string? terminalLine = null; + + string BuildTerminal( + int returnedCount, + bool interrupted, + bool truncated, + int? firstOmittedResultBytes, + int omittedCount, + int omittedRecordCount, + string? recoveryGuidance, + bool includeSelectionAccounting) + => BuildJsonStreamDoneLine( + returnedCount, + totalCount, + jsonOptions, + interrupted, + truncated, + reader, + maxJsonBytes: options.MaxJsonBytes, + firstOmittedResultBytes: firstOmittedResultBytes, + omittedCount: omittedCount, + omittedRecordCount: omittedRecordCount, + appliedLimit: options.Limit, + recoveryGuidance: recoveryGuidance, + totalCountAuthoritative: totalCountAuthoritative, + truncationReason: truncationReason, + selectionReason: selectionReason, + selectionOmittedCount: selectionOmittedCount, + sourceTotal: includeSelectionAccounting ? sourceTotal : null, + sourceTotalAuthoritative: includeSelectionAccounting ? sourceTotalAuthoritative : null, + selectedTotal: includeSelectionAccounting ? selectedTotal : null, + selectorOmittedCount: includeSelectionAccounting ? selectorOmittedCount : null, + limitOmittedCount: includeSelectionAccounting ? limitOmittedCount : null, + selectors: includeSelectionAccounting ? selectors : null); + if (options.MaxJsonBytes.HasValue) { for (var candidate = records.Count; candidate >= 0; candidate--) @@ -55,26 +95,33 @@ private static NdjsonStreamWriteResult WriteNdjsonStream( var candidateReturnedCount = CountResults(records, candidate); var candidateInterrupted = candidate < records.Count; var candidateFirstOmittedBytes = candidateInterrupted ? JsonLineBytes(records[candidate].Line) : (int?)null; - var candidateTerminal = BuildJsonStreamDoneLine( + var candidateRecoveryGuidance = candidateInterrupted + ? "Increase --max-json-bytes or reduce --limit. Pass --allow-partial only when exit code 0 is acceptable for incomplete output." + : limitTruncated ? limitRecoveryGuidance : null; + var candidateTerminal = BuildTerminal( candidateReturnedCount, - totalCount, - jsonOptions, - interrupted: candidateInterrupted, - truncated: limitTruncated || candidateInterrupted, - reader, - maxJsonBytes: options.MaxJsonBytes, - firstOmittedResultBytes: candidateFirstOmittedBytes, - omittedCount: Math.Max(0, totalCount - candidateReturnedCount), - omittedRecordCount: Math.Max(0, records.Count - candidate), - appliedLimit: options.Limit, - recoveryGuidance: candidateInterrupted - ? "Increase --max-json-bytes or reduce --limit. Pass --allow-partial only when exit code 0 is acceptable for incomplete output." - : limitTruncated ? limitRecoveryGuidance : null, - totalCountAuthoritative: totalCountAuthoritative, - truncationReason: truncationReason, - selectionReason: selectionReason, - selectionOmittedCount: selectionOmittedCount); - if (PrefixBytes(records, candidate) + JsonLineBytes(candidateTerminal) > options.MaxJsonBytes.Value) + candidateInterrupted, + limitTruncated || candidateInterrupted, + candidateFirstOmittedBytes, + Math.Max(0, totalCount - candidateReturnedCount), + Math.Max(0, records.Count - candidate), + candidateRecoveryGuidance, + includeSelectionAccounting: true); + var candidatePrefixBytes = PrefixBytes(records, candidate); + if (candidatePrefixBytes + JsonLineBytes(candidateTerminal) > options.MaxJsonBytes.Value + && sourceTotal.HasValue) + { + candidateTerminal = BuildTerminal( + candidateReturnedCount, + candidateInterrupted, + limitTruncated || candidateInterrupted, + candidateFirstOmittedBytes, + Math.Max(0, totalCount - candidateReturnedCount), + Math.Max(0, records.Count - candidate), + candidateRecoveryGuidance, + includeSelectionAccounting: false); + } + if (candidatePrefixBytes + JsonLineBytes(candidateTerminal) > options.MaxJsonBytes.Value) continue; emittedRecords = candidate; @@ -84,23 +131,28 @@ private static NdjsonStreamWriteResult WriteNdjsonStream( if (terminalLine == null) { - var requiredTerminal = BuildJsonStreamDoneLine( - count: 0, + var requiredTerminal = BuildTerminal( + 0, + records.Count > 0, + limitTruncated || records.Count > 0, + records.Count > 0 ? JsonLineBytes(records[0].Line) : null, totalCount, - jsonOptions, - interrupted: records.Count > 0, - truncated: limitTruncated || records.Count > 0, - reader, - maxJsonBytes: options.MaxJsonBytes, - firstOmittedResultBytes: records.Count > 0 ? JsonLineBytes(records[0].Line) : null, - omittedCount: totalCount, - omittedRecordCount: records.Count, - appliedLimit: options.Limit, - recoveryGuidance: "Increase --max-json-bytes so the bounded NDJSON terminal record fits before streaming begins.", - totalCountAuthoritative: totalCountAuthoritative, - truncationReason: truncationReason, - selectionReason: selectionReason, - selectionOmittedCount: selectionOmittedCount); + records.Count, + "Increase --max-json-bytes so the bounded NDJSON terminal record fits before streaming begins.", + includeSelectionAccounting: true); + if (JsonLineBytes(requiredTerminal) > options.MaxJsonBytes.Value + && sourceTotal.HasValue) + { + requiredTerminal = BuildTerminal( + 0, + records.Count > 0, + limitTruncated || records.Count > 0, + records.Count > 0 ? JsonLineBytes(records[0].Line) : null, + totalCount, + records.Count, + "Increase --max-json-bytes so the bounded NDJSON terminal record fits before streaming begins.", + includeSelectionAccounting: false); + } WriteUsageError( $"{commandName} NDJSON terminal record is {JsonLineBytes(requiredTerminal)} bytes and exceeds --max-json-bytes {options.MaxJsonBytes.Value}.", GetUsageLineOrThrow(commandName), @@ -111,21 +163,15 @@ private static NdjsonStreamWriteResult WriteNdjsonStream( else { var returnedCount = CountResults(records, emittedRecords); - terminalLine = BuildJsonStreamDoneLine( + terminalLine = BuildTerminal( returnedCount, - totalCount, - jsonOptions, interrupted: false, - truncated: limitTruncated, - reader, - omittedCount: Math.Max(0, totalCount - returnedCount), + limitTruncated, + firstOmittedResultBytes: null, + Math.Max(0, totalCount - returnedCount), omittedRecordCount: 0, - appliedLimit: options.Limit, - recoveryGuidance: limitTruncated ? limitRecoveryGuidance : null, - totalCountAuthoritative: totalCountAuthoritative, - truncationReason: truncationReason, - selectionReason: selectionReason, - selectionOmittedCount: selectionOmittedCount); + limitTruncated ? limitRecoveryGuidance : null, + includeSelectionAccounting: true); } for (var i = 0; i < emittedRecords; i++) diff --git a/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs b/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs index ebad701ecd..922f15325f 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs @@ -258,6 +258,8 @@ private static JsonObject BuildQueryContextJson(QueryCommandOptions options, Jso query["since"] = options.Since.Value; if (options.CountOnly) query["count"] = true; + if (options.FirstPerFile || options.SampleSize.HasValue) + query["row_selectors"] = BuildSearchRowSelectorContextJson(options); if (options.All) query["all"] = true; if (options.RawFts) @@ -309,6 +311,31 @@ private static JsonObject BuildQueryContextJson(QueryCommandOptions options, Jso return query; } + private static JsonArray BuildSearchRowSelectorContextJson(QueryCommandOptions options) + { + var selectors = new JsonArray(); + if (options.FirstPerFile) + { + selectors.Add(new JsonObject + { + ["mode"] = "first_per_file", + ["applied"] = true, + }); + } + if (options.SampleSize.HasValue) + { + selectors.Add(new JsonObject + { + ["mode"] = "sample", + ["applied"] = true, + ["sample_size"] = options.SampleSize.Value, + ["sample_mode"] = SearchSampleMode, + ["seed"] = SearchSampleSeed, + }); + } + return selectors; + } + private static JsonArray BuildSearchGuardFiltersJson(IReadOnlyList guardFilters) { var filters = new JsonArray(); diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Search.cs b/src/CodeIndex/Cli/QueryCommandRunner.Search.cs index a8918bc98d..5dcafae0e6 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Search.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Search.cs @@ -782,7 +782,13 @@ [new NdjsonOutputRecord(payload, CountsAsResult: false)], "search", limitTruncated: false, "Increase --limit or narrow the query to retrieve the remaining search results.", - totalCountAuthoritative: false); + totalCountAuthoritative: false, + sourceTotal: selection.Selectors.Count > 0 ? selection.SourceTotal : null, + sourceTotalAuthoritative: selection.Selectors.Count > 0 ? selection.SourceTotalAuthoritative : null, + selectedTotal: selection.Selectors.Count > 0 ? selection.SelectedTotal : null, + selectorOmittedCount: selection.Selectors.Count > 0 ? selection.SelectorOmittedCount : null, + limitOmittedCount: selection.Selectors.Count > 0 ? selection.LimitOmittedCount : null, + selectors: selection.Selectors.Count > 0 ? selection.Selectors : null); jsonDoneTerminalLine = stream.TerminalLine; return stream.ExitCode == CommandExitCodes.Success ? ZeroResultExitCode(options) : stream.ExitCode; } @@ -815,6 +821,12 @@ [new NdjsonOutputRecord(payload, CountsAsResult: false)], selection.TruncationReason is "sample" or "first_per_file" ? selection.SelectionOmittedCount : null, + selection.SourceTotal, + selection.SourceTotalAuthoritative, + selection.SelectedTotal, + selection.SelectorOmittedCount, + selection.LimitOmittedCount, + selection.Selectors, options, jsonOptions, ndjsonOptions, @@ -876,6 +888,12 @@ selection.TruncationReason is "sample" or "first_per_file" selection.TruncationReason is "sample" or "first_per_file" ? selection.SelectionOmittedCount : null, + selection.SourceTotal, + selection.SourceTotalAuthoritative, + selection.SelectedTotal, + selection.SelectorOmittedCount, + selection.LimitOmittedCount, + selection.Selectors, options, ndjsonOptions, reader, diff --git a/src/CodeIndex/Cli/QueryCommandRunner.SearchConstants.cs b/src/CodeIndex/Cli/QueryCommandRunner.SearchConstants.cs index ef9200a67b..fa68110655 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.SearchConstants.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.SearchConstants.cs @@ -28,5 +28,7 @@ public static partial class QueryCommandRunner private const int SearchEnvelopeMinCandidates = 200; private const int SearchEnvelopeOverFetchFactor = 50; private const int SearchEnvelopeMaxCandidates = MaxQueryResultLimit; + private const int SearchSampleSeed = 0; + private const string SearchSampleMode = "deterministic_seeded_v1"; private const string SearchFilterNoMatchSentinel = "\0__cdidx_no_match__"; } diff --git a/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs b/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs index d2c81d354e..8dea95d840 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs @@ -1092,6 +1092,8 @@ private static NdjsonStreamWriteResult WriteRecipeSearchResultRows( var selectionOmittedCount = queryResults .Where(query => query.SelectionOmittedCount.HasValue) .Sum(query => query.SelectionOmittedCount!.Value); + var selectors = AggregateSearchRowSelectors(queryResults.SelectMany(query => query.Selectors)); + var hasSelectors = selectors.Count > 0; return WriteNdjsonStream( records, totalCount, @@ -1104,8 +1106,32 @@ private static NdjsonStreamWriteResult WriteRecipeSearchResultRows( totalCountAuthoritative: false, truncationReason: limitTruncated ? "limit" : null, selectionReason: selectionReason, - selectionOmittedCount: selectionReason != null ? selectionOmittedCount : null); - } + selectionOmittedCount: selectionReason != null ? selectionOmittedCount : null, + sourceTotal: hasSelectors ? queryResults.Sum(query => query.SourceTotal) : null, + sourceTotalAuthoritative: hasSelectors + ? queryResults.All(query => query.SourceTotalAuthoritative) + : null, + selectedTotal: hasSelectors ? queryResults.Sum(query => query.SelectedTotal) : null, + selectorOmittedCount: hasSelectors ? queryResults.Sum(query => query.SelectorOmittedCount) : null, + limitOmittedCount: hasSelectors ? queryResults.Sum(query => query.LimitOmittedCount) : null, + selectors: hasSelectors ? selectors : null); + } + + private static List AggregateSearchRowSelectors( + IEnumerable selectors) + => selectors + .GroupBy( + selector => (selector.Mode, selector.SampleSize, selector.SampleMode, selector.Seed)) + .Select(group => new SearchRowSelectorJsonResult( + group.Key.Mode, + group.All(selector => selector.Applied), + group.Sum(selector => selector.InputTotal), + group.Sum(selector => selector.OutputTotal), + group.Sum(selector => selector.OmittedCount), + group.Key.SampleSize, + group.Key.SampleMode, + group.Key.Seed)) + .ToList(); private static JsonObject BuildRecipeSearchResultRow( string recipeName, @@ -1339,7 +1365,15 @@ private static int RunSearchIssueDrafts( BuildSearchRecipeTopFiles(rows), false, null, - rows.Select(row => row.Compact).ToList()); + rows.Select(row => row.Compact).ToList(), + rows.Count, + true, + null, + rows.Count, + rows.Count, + 0, + 0, + []); var drafts = rows.Count == 0 ? [] : new List { ToAdHocSearchIssueDraft(options, queryResult, preflight) }; @@ -1389,9 +1423,10 @@ private static List CollectSearchRecipeQueryR var queryScope = BuildSearchRecipeQueryScope(scope, recipeQuery); var resultLimit = GetSearchRecipeEffectiveResultLimit(options, total); var guardFilters = BuildSearchRecipeGuardFilters(options, recipeQuery); + var fetchLimit = GetSearchRecipeFetchLimit(options, resultLimit); var results = reader.Search( recipeQuery.Query, - GetSearchRecipeFetchLimit(options, resultLimit), + fetchLimit, options.Lang, false, queryScope.PathPatterns, @@ -1408,9 +1443,10 @@ private static List CollectSearchRecipeQueryR guardScope: options.GuardScope, requiredPathPatterns: GetSearchRecipeRequiredPathPatterns(options, recipeQuery), resultRanking: GetSearchRecipeResultRanking(recipeQuery.ResultRanking, resultLimit)); + var sourceTotalAuthoritative = results.Count < fetchLimit; results = ApplySearchRecipeFileRejectQueries(reader, results, options, recipeQuery); var rows = BuildSearchDisplayRows(results, options, exact, recipeQuery.Query, rawFtsOverride: false, recipeQuery: recipeQuery); - var outputSelection = ApplySearchOutputSelection(rows, options, resultLimit); + var outputSelection = ApplySearchOutputSelection(rows, options, resultLimit, sourceTotalAuthoritative); rows = outputSelection.Rows; ApplySearchRecipeAuditClassifications(recipeQuery, rows); var minimumOmitted = Math.Max(0, outputSelection.OriginalCount - rows.Count); @@ -1453,7 +1489,15 @@ private static List CollectSearchRecipeQueryR && rows.Count > 0 ? FormatSearchCursor(rows[^1].Result) : null, - rows.Select(row => row.Compact).ToList())); + rows.Select(row => row.Compact).ToList(), + outputSelection.SourceTotal, + outputSelection.SourceTotalAuthoritative, + outputSelection.SourceTotalAuthoritative ? null : outputSelection.SourceTotal, + outputSelection.SelectedTotal, + outputSelection.Returned, + outputSelection.SelectorOmittedCount, + outputSelection.LimitOmittedCount, + outputSelection.Selectors)); } return queryResults; @@ -1475,9 +1519,10 @@ private static List CollectSearchRecip var queryScope = BuildSearchRecipeQueryScope(scope, recipeQuery); var resultLimit = GetSearchRecipeEffectiveResultLimit(options, total); var guardFilters = BuildSearchRecipeGuardFilters(options, recipeQuery); + var fetchLimit = GetSearchRecipeFetchLimit(options, resultLimit); var results = reader.Search( recipeQuery.Query, - GetSearchRecipeFetchLimit(options, resultLimit), + fetchLimit, options.Lang, false, queryScope.PathPatterns, @@ -1494,9 +1539,10 @@ private static List CollectSearchRecip guardScope: options.GuardScope, requiredPathPatterns: GetSearchRecipeRequiredPathPatterns(options, recipeQuery), resultRanking: GetSearchRecipeResultRanking(recipeQuery.ResultRanking, resultLimit)); + var sourceTotalAuthoritative = results.Count < fetchLimit; results = ApplySearchRecipeFileRejectQueries(reader, results, options, recipeQuery); var rows = BuildSearchDisplayRows(results, options, exact, recipeQuery.Query, recipeQuery: recipeQuery); - var outputSelection = ApplySearchOutputSelection(rows, options, resultLimit); + var outputSelection = ApplySearchOutputSelection(rows, options, resultLimit, sourceTotalAuthoritative); rows = outputSelection.Rows; ApplySearchRecipeAuditClassifications(recipeQuery, rows); var minimumOmitted = Math.Max(0, outputSelection.OriginalCount - rows.Count); @@ -1543,7 +1589,15 @@ private static List CollectSearchRecip row.Result.EndLine, row.Compact.MatchLines, row.Compact.EnclosingSymbolName, - row.Compact.EnclosingSymbolKind)).ToList())); + row.Compact.EnclosingSymbolKind)).ToList(), + outputSelection.SourceTotal, + outputSelection.SourceTotalAuthoritative, + outputSelection.SourceTotalAuthoritative ? null : outputSelection.SourceTotal, + outputSelection.SelectedTotal, + outputSelection.Returned, + outputSelection.SelectorOmittedCount, + outputSelection.LimitOmittedCount, + outputSelection.Selectors)); } return queryResults; @@ -1935,7 +1989,16 @@ private static SearchRecipeRunSummaryJsonResult BuildSearchRecipeRunSummary( queryResults.Any(query => query.Truncated && !string.IsNullOrWhiteSpace(query.NextCursor)), BuildSearchRecipeCursoringHint( queryResults.Any(query => query.Truncated), - queryResults.Any(query => query.Truncated && !string.IsNullOrWhiteSpace(query.NextCursor)))); + queryResults.Any(query => query.Truncated && !string.IsNullOrWhiteSpace(query.NextCursor))), + queryResults.Sum(query => query.SourceTotal), + queryResults.All(query => query.SourceTotalAuthoritative), + queryResults.All(query => query.SourceTotalAuthoritative) + ? null + : queryResults.Sum(query => query.SourceTotal), + queryResults.Sum(query => query.SelectedTotal), + queryResults.Sum(query => query.Returned), + queryResults.Sum(query => query.SelectorOmittedCount), + queryResults.Sum(query => query.LimitOmittedCount)); private static SearchRecipeRunSummaryJsonResult BuildSearchRecipeRunSummary( IReadOnlyList queryResults, @@ -1952,7 +2015,16 @@ private static SearchRecipeRunSummaryJsonResult BuildSearchRecipeRunSummary( queryResults.Any(query => query.Truncated && !string.IsNullOrWhiteSpace(query.NextCursor)), BuildSearchRecipeCursoringHint( queryResults.Any(query => query.Truncated), - queryResults.Any(query => query.Truncated && !string.IsNullOrWhiteSpace(query.NextCursor)))); + queryResults.Any(query => query.Truncated && !string.IsNullOrWhiteSpace(query.NextCursor))), + queryResults.Sum(query => query.SourceTotal), + queryResults.All(query => query.SourceTotalAuthoritative), + queryResults.All(query => query.SourceTotalAuthoritative) + ? null + : queryResults.Sum(query => query.SourceTotal), + queryResults.Sum(query => query.SelectedTotal), + queryResults.Sum(query => query.Returned), + queryResults.Sum(query => query.SelectorOmittedCount), + queryResults.Sum(query => query.LimitOmittedCount)); private static string BuildSearchRecipeCursoringHint(bool hasTruncatedQuery, bool cursoringAvailable) => cursoringAvailable @@ -2329,7 +2401,15 @@ private static SearchIssueDraftJsonResult ToSearchIssueDraft( queryResult.SelectionOmittedCount, queryResult.MinimumOmittedResultCount, queryResult.Truncated, - queryResult.NextCursor), + queryResult.NextCursor, + queryResult.SourceTotal, + queryResult.SourceTotalAuthoritative, + queryResult.SourceTotalLowerBound, + queryResult.SelectedTotal, + queryResult.Returned, + queryResult.SelectorOmittedCount, + queryResult.LimitOmittedCount, + queryResult.Selectors), new SuggestionIssueDraftDuplicatePreflightJsonResult( preflight.Checked, duplicateMatches.Count, @@ -2386,7 +2466,15 @@ private static SearchIssueDraftJsonResult ToAdHocSearchIssueDraft( null, queryResult.MinimumOmittedResultCount, queryResult.Truncated, - queryResult.NextCursor), + queryResult.NextCursor, + queryResult.SourceTotal, + queryResult.SourceTotalAuthoritative, + queryResult.SourceTotalLowerBound, + queryResult.SelectedTotal, + queryResult.Returned, + queryResult.SelectorOmittedCount, + queryResult.LimitOmittedCount, + queryResult.Selectors), new SuggestionIssueDraftDuplicatePreflightJsonResult( preflight.Checked, duplicateMatches.Count, diff --git a/src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs b/src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs index 74010a5f6f..d447f9748f 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs @@ -369,47 +369,48 @@ private static bool IsSupportedSearchAggregationValue(string value) => NormalizeSearchAggregationKey(value) is "file" or "symbol" or "origin" or "return_type" or "subsystem"; private static SearchOutputSelection ApplySearchOutputSelection(List rows, QueryCommandOptions options) - => ApplySearchOutputSelection(rows, options, options.Limit); + => ApplySearchOutputSelection( + rows, + options, + options.Limit, + !options.FirstPerFile && !options.SampleSize.HasValue + || rows.Count < SearchOriginFilterMaxCandidates); private static SearchOutputSelection ApplySearchOutputSelection( List rows, QueryCommandOptions options, - int limit) + int limit, + bool sourceTotalAuthoritative = true) { - var originalCount = rows.Count; - rows = ApplySearchPostSelectors( - rows, - options, - out var firstPerFileTruncated, - out var sampleTruncated); - var postSelectionCount = rows.Count; + var sourceTotal = rows.Count; + var selectors = new List(); + rows = ApplySearchPostSelectors(rows, options, selectors); + var selectedTotal = rows.Count; var limitTruncated = rows.Count > limit; if (limitTruncated) rows = rows.Take(limit).ToList(); - var truncationReason = firstPerFileTruncated - ? "first_per_file" - : sampleTruncated - ? "sample" - : limitTruncated - ? "limit" - : null; + var selectionReason = selectors.FirstOrDefault(selector => selector.OmittedCount > 0)?.Mode; + var truncationReason = selectionReason ?? (limitTruncated ? "limit" : null); return new SearchOutputSelection( rows, - originalCount, - rows.Count < originalCount, + sourceTotal, + selectedTotal, + rows.Count, + Math.Max(0, sourceTotal - selectedTotal), + Math.Max(0, selectedTotal - rows.Count), + sourceTotalAuthoritative, + rows.Count < sourceTotal, limitTruncated, truncationReason, - Math.Max(0, originalCount - postSelectionCount)); + selectors); } private static List ApplySearchPostSelectors( List rows, QueryCommandOptions options, - out bool firstPerFileTruncated, - out bool sampleTruncated) + List selectors) { - firstPerFileTruncated = false; if (options.FirstPerFile) { var beforeFirstPerFile = rows.Count; @@ -417,14 +418,28 @@ private static List ApplySearchPostSelectors( .GroupBy(row => row.Result.Path, StringComparer.Ordinal) .Select(group => group.First()) .ToList(); - firstPerFileTruncated = rows.Count < beforeFirstPerFile; + selectors.Add(new SearchRowSelectorJsonResult( + "first_per_file", + true, + beforeFirstPerFile, + rows.Count, + Math.Max(0, beforeFirstPerFile - rows.Count))); } - sampleTruncated = false; - if (options.SampleSize.HasValue && rows.Count > options.SampleSize.Value) + if (options.SampleSize.HasValue) { - sampleTruncated = true; - rows = SampleSearchRows(rows, options.SampleSize.Value); + var beforeSample = rows.Count; + if (rows.Count > options.SampleSize.Value) + rows = SampleSearchRows(rows, options.SampleSize.Value); + selectors.Add(new SearchRowSelectorJsonResult( + "sample", + true, + beforeSample, + rows.Count, + Math.Max(0, beforeSample - rows.Count), + options.SampleSize.Value, + SearchSampleMode, + SearchSampleSeed)); } return rows; } @@ -436,14 +451,44 @@ private static List SampleSearchRows(List ro if (sampleSize == 1) return [rows[0]]; - var sampled = new List(sampleSize); - var lastIndex = rows.Count - 1; - for (var i = 0; i < sampleSize; i++) + return rows + .Select((row, index) => new + { + Row = row, + Index = index, + Key = ComputeSearchSampleKey(row, index), + }) + .OrderBy(candidate => candidate.Key) + .ThenBy(candidate => candidate.Index) + .Take(sampleSize) + .OrderBy(candidate => candidate.Index) + .Select(candidate => candidate.Row) + .ToList(); + } + + private static ulong ComputeSearchSampleKey(SearchDisplayRow row, int index) + { + const ulong offsetBasis = 14695981039346656037; + const ulong prime = 1099511628211; + var hash = offsetBasis ^ (uint)SearchSampleSeed; + + Add(row.Result.Path); + Add(row.Result.ChunkId.ToString(CultureInfo.InvariantCulture)); + Add(row.Result.StartLine.ToString(CultureInfo.InvariantCulture)); + Add(row.Result.EndLine.ToString(CultureInfo.InvariantCulture)); + Add(index.ToString(CultureInfo.InvariantCulture)); + return hash; + + void Add(string value) { - var index = (int)Math.Round(i * (lastIndex / (double)(sampleSize - 1)), MidpointRounding.AwayFromZero); - sampled.Add(rows[Math.Clamp(index, 0, lastIndex)]); + foreach (var character in value) + { + hash ^= character; + hash *= prime; + } + hash ^= 0xff; + hash *= prime; } - return sampled; } private static int WriteGroupedSearchResults( @@ -545,6 +590,12 @@ private static int WriteProjectedSearchResults( string? truncationReason, string? selectionReason, int? selectionOmittedCount, + int sourceTotal, + bool sourceTotalAuthoritative, + int selectedTotal, + int selectorOmittedCount, + int limitOmittedCount, + List selectors, QueryCommandOptions options, JsonSerializerOptions jsonOptions, JsonSerializerOptions ndjsonOptions, @@ -586,7 +637,13 @@ private static int WriteProjectedSearchResults( totalCountAuthoritative: false, truncationReason: truncationReason, selectionReason: selectionReason, - selectionOmittedCount: selectionOmittedCount); + selectionOmittedCount: selectionOmittedCount, + sourceTotal: selectors.Count > 0 ? sourceTotal : null, + sourceTotalAuthoritative: selectors.Count > 0 ? sourceTotalAuthoritative : null, + selectedTotal: selectors.Count > 0 ? selectedTotal : null, + selectorOmittedCount: selectors.Count > 0 ? selectorOmittedCount : null, + limitOmittedCount: selectors.Count > 0 ? limitOmittedCount : null, + selectors: selectors.Count > 0 ? selectors : null); terminalLine = stream.TerminalLine; return stream.ExitCode; } @@ -653,6 +710,12 @@ private static int WriteSearchNdjsonResults( string? truncationReason, string? selectionReason, int? selectionOmittedCount, + int sourceTotal, + bool sourceTotalAuthoritative, + int selectedTotal, + int selectorOmittedCount, + int limitOmittedCount, + List selectors, QueryCommandOptions options, JsonSerializerOptions ndjsonOptions, DbReader reader, @@ -674,7 +737,13 @@ private static int WriteSearchNdjsonResults( totalCountAuthoritative: false, truncationReason: truncationReason, selectionReason: selectionReason, - selectionOmittedCount: selectionOmittedCount); + selectionOmittedCount: selectionOmittedCount, + sourceTotal: selectors.Count > 0 ? sourceTotal : null, + sourceTotalAuthoritative: selectors.Count > 0 ? sourceTotalAuthoritative : null, + selectedTotal: selectors.Count > 0 ? selectedTotal : null, + selectorOmittedCount: selectors.Count > 0 ? selectorOmittedCount : null, + limitOmittedCount: selectors.Count > 0 ? limitOmittedCount : null, + selectors: selectors.Count > 0 ? selectors : null); terminalLine = stream.TerminalLine; return stream.ExitCode; } @@ -745,11 +814,20 @@ private static void AttachSearchNextSteps(CompactSearchResult[] results, QueryCo private sealed record SearchOutputSelection( List Rows, - int OriginalCount, + int SourceTotal, + int SelectedTotal, + int Returned, + int SelectorOmittedCount, + int LimitOmittedCount, + bool SourceTotalAuthoritative, bool Truncated, bool LimitTruncated, string? TruncationReason, - int SelectionOmittedCount); + List Selectors) + { + public int OriginalCount => SourceTotal; + public int SelectionOmittedCount => SelectorOmittedCount; + } private sealed record SearchGroupOutputSelection( List Groups, @@ -1115,8 +1193,7 @@ private static List ReadSearchDisplayRows(DbReader reader, Que var selectedRows = ApplySearchPostSelectors( boundedRows, options, - out _, - out _); + []); JsonEnvelopeWrapper.ReportBoundedResponseTotal( "search", selectedRows.Count, @@ -1218,8 +1295,7 @@ private static int GetSearchDisplayCandidateLimit(QueryCommandOptions options) return requested; if (!options.FirstPerFile && !options.SampleSize.HasValue) return requested == int.MaxValue ? requested : requested + 1; - var sampleTarget = Math.Max(requested, options.SampleSize ?? requested); - return Math.Min(SearchOriginFilterMaxCandidates, Math.Max(requested + 1, sampleTarget * SearchOriginFilterOverFetchFactor)); + return SearchOriginFilterMaxCandidates; } private static List ReadSearchResults(DbReader reader, QueryCommandOptions options, bool exact, int limit, SearchCursor? cursor = null, int? guardRequestedLimit = null) @@ -1569,7 +1645,13 @@ private static string BuildJsonStreamDoneLine( bool totalCountAuthoritative = true, string? truncationReason = null, string? selectionReason = null, - int? selectionOmittedCount = null) + int? selectionOmittedCount = null, + int? sourceTotal = null, + bool? sourceTotalAuthoritative = null, + int? selectedTotal = null, + int? selectorOmittedCount = null, + int? limitOmittedCount = null, + List? selectors = null) { var includeDiagnostics = HasReadOnlyFallbackDiagnostics(reader); return JsonSerializer.Serialize( @@ -1585,6 +1667,14 @@ private static string BuildJsonStreamDoneLine( TotalCountLowerBound: totalCountAuthoritative ? null : totalCount, SelectionReason: selectionReason, SelectionOmittedCount: selectionOmittedCount, + SourceTotal: sourceTotal, + SourceTotalAuthoritative: sourceTotalAuthoritative, + SourceTotalLowerBound: sourceTotalAuthoritative == false ? sourceTotal : null, + SelectedTotal: selectedTotal, + Returned: sourceTotal.HasValue ? count : null, + SelectorOmittedCount: selectorOmittedCount, + LimitOmittedCount: limitOmittedCount, + Selectors: selectors, InterruptionReason: interrupted ? "max_json_bytes_exceeded" : null, TruncationReason: interrupted ? "max_json_bytes_exceeded" : truncated ? truncationReason ?? "limit" : null, AppliedLimit: truncated ? appliedLimit : null, diff --git a/src/CodeIndex/Cli/SearchAuditRecipes.cs b/src/CodeIndex/Cli/SearchAuditRecipes.cs index e3b914b543..6dc4a1aa60 100644 --- a/src/CodeIndex/Cli/SearchAuditRecipes.cs +++ b/src/CodeIndex/Cli/SearchAuditRecipes.cs @@ -4335,13 +4335,38 @@ internal sealed record SearchRecipeRunSummaryJsonResult( [property: JsonPropertyName("minimum_omitted_result_count")] int MinimumOmittedResultCount, [property: JsonPropertyName("query_freshness")] SearchRecipeQueryFreshnessJsonResult QueryFreshness, [property: JsonPropertyName("cursoring_available")] bool CursoringAvailable, - [property: JsonPropertyName("cursoring_hint")] string CursoringHint); + [property: JsonPropertyName("cursoring_hint")] string CursoringHint, + [property: JsonPropertyName("source_total")] int SourceTotal, + [property: JsonPropertyName("source_total_authoritative")] bool SourceTotalAuthoritative, + [property: JsonPropertyName("source_total_lower_bound")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + int? SourceTotalLowerBound, + [property: JsonPropertyName("selected_total")] int SelectedTotal, + [property: JsonPropertyName("returned")] int Returned, + [property: JsonPropertyName("selector_omitted_count")] int SelectorOmittedCount, + [property: JsonPropertyName("limit_omitted_count")] int LimitOmittedCount); internal sealed record SearchRecipeQueryFreshnessJsonResult( [property: JsonPropertyName("positive_evidence_query_count")] int PositiveEvidenceQueryCount, [property: JsonPropertyName("zero_result_query_count")] int ZeroResultQueryCount, [property: JsonPropertyName("stale_query_names")] List StaleQueryNames); +internal sealed record SearchRowSelectorJsonResult( + [property: JsonPropertyName("mode")] string Mode, + [property: JsonPropertyName("applied")] bool Applied, + [property: JsonPropertyName("input_total")] int InputTotal, + [property: JsonPropertyName("output_total")] int OutputTotal, + [property: JsonPropertyName("omitted_count")] int OmittedCount, + [property: JsonPropertyName("sample_size")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + int? SampleSize = null, + [property: JsonPropertyName("sample_mode")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + string? SampleMode = null, + [property: JsonPropertyName("seed")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + int? Seed = null); + internal sealed record SearchNamedBatchRunJsonResult( [property: JsonPropertyName("api_version")] string ApiVersion, [property: JsonPropertyName("query_count")] int QueryCount, @@ -4401,7 +4426,17 @@ internal sealed record SearchRecipeQueryResultJsonResult( [property: JsonPropertyName("top_files")] List TopFiles, [property: JsonPropertyName("truncated")] bool Truncated, [property: JsonPropertyName("next_cursor")] string? NextCursor, - [property: JsonPropertyName("results")] List Results); + [property: JsonPropertyName("results")] List Results, + [property: JsonPropertyName("source_total")] int SourceTotal, + [property: JsonPropertyName("source_total_authoritative")] bool SourceTotalAuthoritative, + [property: JsonPropertyName("source_total_lower_bound")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + int? SourceTotalLowerBound, + [property: JsonPropertyName("selected_total")] int SelectedTotal, + [property: JsonPropertyName("returned")] int Returned, + [property: JsonPropertyName("selector_omitted_count")] int SelectorOmittedCount, + [property: JsonPropertyName("limit_omitted_count")] int LimitOmittedCount, + [property: JsonPropertyName("selectors")] List Selectors); internal sealed record SearchNamedBatchCountSummaryRunJsonResult( [property: JsonPropertyName("api_version")] string ApiVersion, @@ -4528,7 +4563,17 @@ internal sealed record SearchRecipeCompactQueryResultJsonResult( [property: JsonPropertyName("top_files")] List TopFiles, [property: JsonPropertyName("truncated")] bool Truncated, [property: JsonPropertyName("next_cursor")] string? NextCursor, - [property: JsonPropertyName("results")] List Results); + [property: JsonPropertyName("results")] List Results, + [property: JsonPropertyName("source_total")] int SourceTotal, + [property: JsonPropertyName("source_total_authoritative")] bool SourceTotalAuthoritative, + [property: JsonPropertyName("source_total_lower_bound")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + int? SourceTotalLowerBound, + [property: JsonPropertyName("selected_total")] int SelectedTotal, + [property: JsonPropertyName("returned")] int Returned, + [property: JsonPropertyName("selector_omitted_count")] int SelectorOmittedCount, + [property: JsonPropertyName("limit_omitted_count")] int LimitOmittedCount, + [property: JsonPropertyName("selectors")] List Selectors); internal sealed record SearchRecipeTopFileJsonResult( [property: JsonPropertyName("path")] string Path, @@ -4601,7 +4646,17 @@ internal sealed record SearchIssueDraftSourceJsonResult( [property: JsonPropertyName("minimum_omitted_result_count")] int MinimumOmittedResultCount, [property: JsonPropertyName("truncated")] bool Truncated, [property: JsonPropertyName("next_cursor")] - [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? NextCursor); + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? NextCursor, + [property: JsonPropertyName("source_total")] int SourceTotal, + [property: JsonPropertyName("source_total_authoritative")] bool SourceTotalAuthoritative, + [property: JsonPropertyName("source_total_lower_bound")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + int? SourceTotalLowerBound, + [property: JsonPropertyName("selected_total")] int SelectedTotal, + [property: JsonPropertyName("returned")] int Returned, + [property: JsonPropertyName("selector_omitted_count")] int SelectorOmittedCount, + [property: JsonPropertyName("limit_omitted_count")] int LimitOmittedCount, + [property: JsonPropertyName("selectors")] List Selectors); internal sealed record SearchRecipeScopeJsonResult( [property: JsonPropertyName("name")] string Name, diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs index 7f1ba29f76..8593c3e7dc 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs @@ -7203,7 +7203,13 @@ public void RunSearch_RecipeRowSelectionAppliesToJsonCompactAndNdjson_Issue4759( Assert.Equal(1, sampleSummary.GetProperty("emitted_result_count").GetInt32()); Assert.Equal(0, sampleSummary.GetProperty("truncated_query_count").GetInt32()); Assert.Equal(4, sampleSummary.GetProperty("minimum_omitted_result_count").GetInt32()); + Assert.Equal(5, sampleSummary.GetProperty("source_total").GetInt32()); + Assert.Equal(1, sampleSummary.GetProperty("selected_total").GetInt32()); + Assert.Equal(1, sampleSummary.GetProperty("returned").GetInt32()); + Assert.Equal(4, sampleSummary.GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(0, sampleSummary.GetProperty("limit_omitted_count").GetInt32()); Assert.False(sampleSummary.GetProperty("cursoring_available").GetBoolean()); + Assert.True(sampleSummary.GetProperty("source_total_authoritative").GetBoolean()); var sampleQuery = Assert.Single(sampleDocument.RootElement.GetProperty("queries").EnumerateArray()); Assert.Equal(1, sampleQuery.GetProperty("count").GetInt32()); Assert.Equal(1, sampleQuery.GetProperty("emitted_count").GetInt32()); @@ -7211,6 +7217,21 @@ public void RunSearch_RecipeRowSelectionAppliesToJsonCompactAndNdjson_Issue4759( Assert.Equal(4, sampleQuery.GetProperty("omitted_count").GetInt32()); Assert.Equal("sample", sampleQuery.GetProperty("selection_reason").GetString()); Assert.Equal(4, sampleQuery.GetProperty("selection_omitted_count").GetInt32()); + Assert.Equal(5, sampleQuery.GetProperty("source_total").GetInt32()); + Assert.True(sampleQuery.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(1, sampleQuery.GetProperty("selected_total").GetInt32()); + Assert.Equal(1, sampleQuery.GetProperty("returned").GetInt32()); + Assert.Equal(4, sampleQuery.GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(0, sampleQuery.GetProperty("limit_omitted_count").GetInt32()); + var sampleSelector = Assert.Single(sampleQuery.GetProperty("selectors").EnumerateArray()); + Assert.Equal("sample", sampleSelector.GetProperty("mode").GetString()); + Assert.True(sampleSelector.GetProperty("applied").GetBoolean()); + Assert.Equal(5, sampleSelector.GetProperty("input_total").GetInt32()); + Assert.Equal(1, sampleSelector.GetProperty("output_total").GetInt32()); + Assert.Equal(4, sampleSelector.GetProperty("omitted_count").GetInt32()); + Assert.Equal(1, sampleSelector.GetProperty("sample_size").GetInt32()); + Assert.Equal("deterministic_seeded_v1", sampleSelector.GetProperty("sample_mode").GetString()); + Assert.Equal(0, sampleSelector.GetProperty("seed").GetInt32()); Assert.False(sampleQuery.GetProperty("truncated").GetBoolean()); Assert.Equal(JsonValueKind.Null, sampleQuery.GetProperty("next_cursor").ValueKind); Assert.Single(sampleQuery.GetProperty("results").EnumerateArray()); @@ -7249,6 +7270,13 @@ public void RunSearch_RecipeRowSelectionAppliesToJsonCompactAndNdjson_Issue4759( Assert.Equal(5, compactQuery.GetProperty("minimum_matched_count").GetInt32()); Assert.Equal("sample", compactQuery.GetProperty("selection_reason").GetString()); Assert.Equal(4, compactQuery.GetProperty("selection_omitted_count").GetInt32()); + Assert.Equal(5, compactQuery.GetProperty("source_total").GetInt32()); + Assert.True(compactQuery.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(1, compactQuery.GetProperty("selected_total").GetInt32()); + Assert.Equal(1, compactQuery.GetProperty("returned").GetInt32()); + Assert.Equal(4, compactQuery.GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(0, compactQuery.GetProperty("limit_omitted_count").GetInt32()); + Assert.Single(compactQuery.GetProperty("selectors").EnumerateArray()); Assert.False(compactQuery.GetProperty("truncated").GetBoolean()); var compactNextCommand = Assert.Single(compactDocument.RootElement.GetProperty("next_commands").EnumerateArray()).GetString()!; Assert.Contains("--sample 1", compactNextCommand, StringComparison.Ordinal); @@ -7285,6 +7313,13 @@ public void RunSearch_RecipeRowSelectionAppliesToJsonCompactAndNdjson_Issue4759( Assert.Contains("--sample 1", draft.GetProperty("body").GetString(), StringComparison.Ordinal); Assert.Equal("sample", draft.GetProperty("source").GetProperty("selection_reason").GetString()); Assert.Equal(4, draft.GetProperty("source").GetProperty("selection_omitted_count").GetInt32()); + Assert.Equal(5, draft.GetProperty("source").GetProperty("source_total").GetInt32()); + Assert.True(draft.GetProperty("source").GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(1, draft.GetProperty("source").GetProperty("selected_total").GetInt32()); + Assert.Equal(1, draft.GetProperty("source").GetProperty("returned").GetInt32()); + Assert.Equal(4, draft.GetProperty("source").GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(0, draft.GetProperty("source").GetProperty("limit_omitted_count").GetInt32()); + Assert.Single(draft.GetProperty("source").GetProperty("selectors").EnumerateArray()); var (ndjsonExitCode, ndjsonStdout, ndjsonStderr) = CaptureConsole(() => QueryCommandRunner.RunSearch( ["--recipe", "risky-code/raw-diagnostic-echo", "--db", dbPath, "--json=ndjson", "--sample", "1", "--limit", "20"], @@ -7301,6 +7336,13 @@ public void RunSearch_RecipeRowSelectionAppliesToJsonCompactAndNdjson_Issue4759( Assert.Equal(5, ndjsonTerminal.RootElement.GetProperty("total_count_lower_bound").GetInt32()); Assert.Equal("sample", ndjsonTerminal.RootElement.GetProperty("selection_reason").GetString()); Assert.Equal(4, ndjsonTerminal.RootElement.GetProperty("selection_omitted_count").GetInt32()); + Assert.Equal(5, ndjsonTerminal.RootElement.GetProperty("source_total").GetInt32()); + Assert.True(ndjsonTerminal.RootElement.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(1, ndjsonTerminal.RootElement.GetProperty("selected_total").GetInt32()); + Assert.Equal(1, ndjsonTerminal.RootElement.GetProperty("returned").GetInt32()); + Assert.Equal(4, ndjsonTerminal.RootElement.GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(0, ndjsonTerminal.RootElement.GetProperty("limit_omitted_count").GetInt32()); + Assert.Single(ndjsonTerminal.RootElement.GetProperty("selectors").EnumerateArray()); Assert.False(ndjsonTerminal.RootElement.GetProperty("truncated").GetBoolean()); Assert.False(ndjsonTerminal.RootElement.GetProperty("has_more").GetBoolean()); } @@ -7310,6 +7352,158 @@ public void RunSearch_RecipeRowSelectionAppliesToJsonCompactAndNdjson_Issue4759( } } + [Fact] + public void RunSearch_RecipeSelectionAccountingReportsSourceSelectorsAndLimit_Issue4843() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_search_recipe_selection_accounting"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + for (var fileIndex = 0; fileIndex < 6; fileIndex++) + { + var path = $"src/selection-{fileIndex}.cs"; + TestProjectHelper.InsertIndexedFile(dbPath, path, "csharp", "Console.WriteLine(ex.Message);\n"); + ReplaceIndexedChunks( + dbPath, + path, + [.. Enumerable.Range(0, 21).Select(chunkIndex => new ChunkRecord + { + ChunkIndex = chunkIndex, + StartLine = (chunkIndex * 10) + 1, + EndLine = (chunkIndex * 10) + 5, + Content = $"Console.WriteLine(ex.Message); // file {fileIndex}, chunk {chunkIndex}\n" + })]); + } + + string[] sampleArgs = + [ + "--recipe", "risky-code/raw-diagnostic-echo", + "--db", dbPath, + "--json", + "--sample", "5", + "--limit", "20", + ]; + var (sampleExitCode, sampleStdout, sampleStderr) = CaptureConsole(() => + QueryCommandRunner.RunSearch(sampleArgs, _jsonOptions)); + var (repeatExitCode, repeatStdout, repeatStderr) = CaptureConsole(() => + QueryCommandRunner.RunSearch(sampleArgs, _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, sampleExitCode); + Assert.Equal(CommandExitCodes.Success, repeatExitCode); + Assert.Equal(string.Empty, sampleStderr); + Assert.Equal(string.Empty, repeatStderr); + using var sampleDocument = ParseJsonOutput(sampleStdout); + using var repeatDocument = ParseJsonOutput(repeatStdout); + var sampleQuery = Assert.Single(sampleDocument.RootElement.GetProperty("queries").EnumerateArray()); + var repeatQuery = Assert.Single(repeatDocument.RootElement.GetProperty("queries").EnumerateArray()); + + Assert.Equal(126, sampleQuery.GetProperty("source_total").GetInt32()); + Assert.True(sampleQuery.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(5, sampleQuery.GetProperty("selected_total").GetInt32()); + Assert.Equal(5, sampleQuery.GetProperty("returned").GetInt32()); + Assert.Equal(121, sampleQuery.GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(0, sampleQuery.GetProperty("limit_omitted_count").GetInt32()); + var sampleSelector = Assert.Single(sampleQuery.GetProperty("selectors").EnumerateArray()); + Assert.Equal(126, sampleSelector.GetProperty("input_total").GetInt32()); + Assert.Equal(5, sampleSelector.GetProperty("output_total").GetInt32()); + Assert.Equal(121, sampleSelector.GetProperty("omitted_count").GetInt32()); + Assert.Equal( + sampleQuery.GetProperty("results").GetRawText(), + repeatQuery.GetProperty("results").GetRawText()); + + var (combinedExitCode, combinedStdout, combinedStderr) = CaptureConsole(() => + QueryCommandRunner.RunSearch( + [ + "--recipe", "risky-code/raw-diagnostic-echo", + "--db", dbPath, + "--format", "compact", + "--first-per-file", + "--sample", "5", + "--limit", "3", + ], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, combinedExitCode); + Assert.Equal(string.Empty, combinedStderr); + using var combinedDocument = ParseJsonOutput(combinedStdout); + var combinedQuery = Assert.Single(combinedDocument.RootElement.GetProperty("queries").EnumerateArray()); + Assert.Equal(126, combinedQuery.GetProperty("source_total").GetInt32()); + Assert.Equal(5, combinedQuery.GetProperty("selected_total").GetInt32()); + Assert.Equal(3, combinedQuery.GetProperty("returned").GetInt32()); + Assert.Equal(121, combinedQuery.GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(2, combinedQuery.GetProperty("limit_omitted_count").GetInt32()); + var combinedSelectors = combinedQuery.GetProperty("selectors").EnumerateArray().ToArray(); + Assert.Equal(2, combinedSelectors.Length); + Assert.Equal("first_per_file", combinedSelectors[0].GetProperty("mode").GetString()); + Assert.Equal(126, combinedSelectors[0].GetProperty("input_total").GetInt32()); + Assert.Equal(6, combinedSelectors[0].GetProperty("output_total").GetInt32()); + Assert.Equal(120, combinedSelectors[0].GetProperty("omitted_count").GetInt32()); + Assert.Equal("sample", combinedSelectors[1].GetProperty("mode").GetString()); + Assert.Equal(6, combinedSelectors[1].GetProperty("input_total").GetInt32()); + Assert.Equal(5, combinedSelectors[1].GetProperty("output_total").GetInt32()); + Assert.Equal(1, combinedSelectors[1].GetProperty("omitted_count").GetInt32()); + + var (emptyExitCode, emptyStdout, emptyStderr) = CaptureConsole(() => + QueryCommandRunner.RunSearch( + [ + "--recipe", "risky-code/raw-diagnostic-echo", + "--db", dbPath, + "--json", + "--sample", "5", + "--path", "missing/**", + ], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, emptyExitCode); + Assert.Equal(string.Empty, emptyStderr); + using var emptyDocument = ParseJsonOutput(emptyStdout); + var emptyQuery = Assert.Single(emptyDocument.RootElement.GetProperty("queries").EnumerateArray()); + Assert.Equal(0, emptyQuery.GetProperty("source_total").GetInt32()); + Assert.Equal(0, emptyQuery.GetProperty("selected_total").GetInt32()); + Assert.Equal(0, emptyQuery.GetProperty("returned").GetInt32()); + Assert.Equal(0, emptyQuery.GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(0, emptyQuery.GetProperty("limit_omitted_count").GetInt32()); + var emptySelector = Assert.Single(emptyQuery.GetProperty("selectors").EnumerateArray()); + Assert.True(emptySelector.GetProperty("applied").GetBoolean()); + Assert.Equal(0, emptySelector.GetProperty("omitted_count").GetInt32()); + + var (contextExitCode, contextStdout, contextStderr) = CaptureConsole(() => + QueryCommandRunner.RunSearch( + [ + "Issue4843DefinitelyMissing", + "--db", dbPath, + "--json", + "--first-per-file", + "--sample", "5", + ], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, contextExitCode); + Assert.Equal(string.Empty, contextStderr); + var contextLines = contextStdout.Split( + '\n', + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + using var contextDocument = JsonDocument.Parse(contextLines[0]); + var contextSelectors = contextDocument.RootElement + .GetProperty("query_context") + .GetProperty("row_selectors") + .EnumerateArray() + .ToArray(); + Assert.Equal(2, contextSelectors.Length); + Assert.Equal("first_per_file", contextSelectors[0].GetProperty("mode").GetString()); + Assert.True(contextSelectors[0].GetProperty("applied").GetBoolean()); + Assert.Equal("sample", contextSelectors[1].GetProperty("mode").GetString()); + Assert.True(contextSelectors[1].GetProperty("applied").GetBoolean()); + Assert.Equal(5, contextSelectors[1].GetProperty("sample_size").GetInt32()); + Assert.Equal("deterministic_seeded_v1", contextSelectors[1].GetProperty("sample_mode").GetString()); + Assert.Equal(0, contextSelectors[1].GetProperty("seed").GetInt32()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunSearch_RecipeSampleFetchesEnoughCandidatesBeforeLimit_Issue4759() { @@ -7342,8 +7536,29 @@ public void RunSearch_RecipeSampleFetchesEnoughCandidatesBeforeLimit_Issue4759() Assert.Equal("sample", query.GetProperty("selection_reason").GetString()); Assert.Equal(1, query.GetProperty("selection_omitted_count").GetInt32()); Assert.Equal(199, query.GetProperty("omitted_count").GetInt32()); + Assert.Equal(201, query.GetProperty("source_total").GetInt32()); + Assert.Equal(200, query.GetProperty("selected_total").GetInt32()); + Assert.Equal(2, query.GetProperty("returned").GetInt32()); + Assert.Equal(1, query.GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(198, query.GetProperty("limit_omitted_count").GetInt32()); Assert.True(query.GetProperty("truncated").GetBoolean()); Assert.Equal(JsonValueKind.Null, query.GetProperty("next_cursor").ValueKind); + + var (boundedExitCode, boundedStdout, boundedStderr) = CaptureConsole(() => QueryCommandRunner.RunSearch( + ["--recipe", "risky-code/raw-diagnostic-echo", "--db", dbPath, "--json", "--sample", "1", "--limit", "1"], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, boundedExitCode); + Assert.Equal(string.Empty, boundedStderr); + using var boundedDocument = ParseJsonOutput(boundedStdout); + var boundedQuery = Assert.Single(boundedDocument.RootElement.GetProperty("queries").EnumerateArray()); + Assert.Equal(200, boundedQuery.GetProperty("source_total").GetInt32()); + Assert.False(boundedQuery.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(200, boundedQuery.GetProperty("source_total_lower_bound").GetInt32()); + Assert.Equal(1, boundedQuery.GetProperty("selected_total").GetInt32()); + Assert.Equal(1, boundedQuery.GetProperty("returned").GetInt32()); + Assert.Equal(199, boundedQuery.GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(0, boundedQuery.GetProperty("limit_omitted_count").GetInt32()); } finally { @@ -7374,6 +7589,34 @@ public void RunSearch_RecipeRejectsRowSelectionForNonRowOutputs_Issue4759() } } + [Fact] + public void RunAudit_CountRejectsRowSelectorsBeforeExecution_Issue4843() + { + foreach (var selectorArgs in new[] + { + new[] { "--sample", "5" }, + new[] { "--first-per-file" }, + }) + { + var (exitCode, stdout, stderr) = CaptureConsole(() => ProgramRunner.Run( + [ + "audit", + "risky-code/raw-diagnostic-echo", + "--format", "count", + .. selectorArgs, + ], + _jsonOptions, + "test")); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stdout); + Assert.Contains( + "recipe row-selection controls cannot be combined with count", + stderr, + StringComparison.Ordinal); + } + } + [Fact] public void RunSearch_RecipeCursorRequiresSingleChildQuery_Issue3392() { From de587c0bc4b395217ba38b3030dcc01c03f7600d Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 00:57:16 +0900 Subject: [PATCH 2/3] Fix selector accounting review findings (#4843) --- DEVELOPER_GUIDE.md | 4 +- USER_GUIDE.md | 50 ++++-- changelog.d/unreleased/4843.fixed.md | 4 +- .../Cli/QueryCommandRunner.Search.cs | 74 +++++++- .../Cli/QueryCommandRunner.SearchRecipes.cs | 28 ++- .../Cli/QueryCommandRunner.SearchResults.cs | 101 +++++++++-- .../QueryCommandRunnerSearchTests.cs | 166 +++++++++++++++++- 7 files changed, 375 insertions(+), 52 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 59c1c67fc5..231256b454 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1555,7 +1555,7 @@ access. | Bounded high-volume responses | `search`, `definition`, `find`, `status`, `hotspots`, `references`, `callers`, `callees`, `symbols`, `files`, `languages`, `impact`, and `map` accept shared bounded-response controls where their schema exposes them. Newly emitted opaque `--cursor ` values bind the offset to the command/query/filter selection and index generation; legacy `response:v1::` cursors remain accepted for transition. Reuse with changed selection or generation fails with restart-required guidance. `search --format compact`, `symbols --format compact`, and `files --format compact` auto-select the bounded contract, while `search --json=array --json-envelope` provides the opt-in array envelope and `languages --json` selects it when paging or `--max-json-bytes` is requested. Existing compact roots and location rows remain compatible while adding shared metadata. Metadata reports `returned_count`, authoritative `total_count` where available, `omitted_count`, `remaining_count`, `cursor_offset`, `page_limit`, `has_more`, `next_cursor`, `result_stable_at`, `pagination_window_limit`, and `pagination_window_exhausted`. The safety window is 10,000 rows; exhaustion suppresses `next_cursor` rather than returning a cursor that the next request would reject. Pageable commands pass the cursor offset into their database/scan layer instead of serializing an `offset + limit` prefix. `find --all` partial scans encode the next path/line in the opaque cursor so replay continues after the last scanned line. `hotspots` and `impact` page their active primary nested collection as `results`, identify it with `metadata.primary_collection`, and retain scalar/container evidence in `metadata.response_context`; dotted fields such as `callers.path,callers.depth` select that collection and project its rows. The final newline is included in `--max-json-bytes`, and trailing whole rows are removed until the complete envelope fits. `definition` remains metadata-only by default; explicit `--body` content is retained for `body`, `body_content`, or `all`, and suppressed when the projection excludes it. `map --sections` remains its section-level projection, while dotted bounded fields page a selected array section with section-specific totals and scalar projections skip unused ranked arrays. | | Bounded response edge cases | `impact` applies the cursor offset only to the selected nested collection so definition pages do not repeat or alter caller/fallback mode. Plain `map --compact` preserves its established section arrays and truncation payload; a collection projection is rejected when `--summary-only` or an excluding `--sections` filter would remove it. Explicit definition body fields override compact defaults. Profile and verbose records are moved into `metadata.stream_control_records`, and parser/capture failures emit an error envelope only when it fits the active hard byte cap. | | `--count --json` envelope | Count-only JSON for `search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `impact`, and `unused` is a single automation-oriented object. It always includes `count`, applied `query_context`, freshness metadata (`indexed_file_count`, `indexed_at`, `freshness_available`), and trust flags `degraded` / `authoritative_count`; commands with matched-file totals also include `files` and the older `file_count` compatibility alias. `file_count` carries the same value as `files`, remains for compatibility, and is not scheduled for removal before the next major release. `unused --count --json` also includes `returned_bucket_counts`, `returned_contract_domain_counts`, and `summary.by_bucket` / `summary.by_confidence` / `summary.by_contract_domain`. `authoritative_count=false` means a readiness or graph/exact trust signal made the count non-authoritative, while the freshness fields describe the indexed snapshot used for the count. | -| Recipe row selection | Row-producing recipe paths share `ApplySearchOutputSelection`: `--first-per-file` and fixed-seed deterministic `--sample` run before the effective per-query / remaining total limit. Sample fetch envelopes are sized from at least the requested sample target. Aggregate/compact query DTOs, run summaries, issue-draft source DTOs, and NDJSON terminals expose `source_total`, `selected_total`, `returned`, `selector_omitted_count`, and `limit_omitted_count`; `source_total_authoritative` / `source_total_lower_bound` distinguish complete populations from bounded observations. Their ordered `selectors` entries preserve each stage's input/output/omission counts plus sample size, mode, and seed, while nullable `selection_reason` / `selection_omitted_count` remain compatibility summaries. Search `query_context.row_selectors` records the applied selector configuration. Selection-only omission updates matched/omitted lower bounds without setting `truncated`, `has_more`, or `next_cursor`. When a later limit truncates selected rows, limit truncation remains visible but `next_cursor` is suppressed because raw database cursors cannot preserve selector state; incoming `--cursor` values are rejected with either selector for the same reason. Generated compact and issue-draft replay commands retain the selector. Count, aggregation, and summary-only compact shapes reject `--first-per-file` / `--sample`, while every recipe shape rejects grouped-only `--per-file-limit`. | +| Search row selection | Row-producing plain-search and recipe paths share `ApplySearchOutputSelection`: `--first-per-file` and fixed-seed deterministic `--sample` run before the effective per-query / remaining total limit. Sample fetch envelopes are sized from at least the requested sample target. Aggregate/compact query DTOs, plain compact roots, run summaries, issue-draft source DTOs, NDJSON terminals, and bounded array-envelope stream terminals expose `source_total`, `selected_total`, `returned`, `selector_omitted_count`, and `limit_omitted_count`; `source_total_authoritative` / `source_total_lower_bound` distinguish complete populations from bounded observations. Guard filters, origin/facet post-filters, exhausted candidate windows, and recipe file-reject post-filters force lower-bound authority. Their ordered `selectors` entries preserve each stage's input/output/omission counts plus sample size, mode, and seed, while nullable `selection_reason` / `selection_omitted_count` remain compatibility summaries. Bounded plain-search selection is computed once and its selected page is reused by compact/envelope serialization. Search `query_context.row_selectors` records the applied selector configuration. Selection-only omission updates matched/omitted lower bounds without setting `truncated`, `has_more`, or `next_cursor`. When a later limit truncates selected rows, limit truncation remains visible but `next_cursor` is suppressed because raw database cursors cannot preserve selector state; incoming `--cursor` values are rejected with either selector for the same reason. Generated compact and issue-draft replay commands retain the selector. Count, aggregation, named-query, recipe-list, results-only, metadata-free array, unsupported formatted, and summary-only compact shapes reject `--first-per-file` / `--sample`, while every recipe shape rejects grouped-only `--per-file-limit`. | | Ad-hoc issue-draft selection | `search --format issue-drafts` reads the complete filtered ad-hoc population, then applies `--first-per-file`, deterministic `--sample`, and `min(--limit, --total-limit)` in that order. Guarded searches retain their finite candidate inspection contract: `source_total_count` is omitted, `source_minimum_count` reports the observed lower bound, `source_total_count_authoritative=false`, `source_fetch_limit` reports the bounded fetch, and `truncated=true` preserves incomplete-population state. Existing `result_count`, `result_limit`, `omitted_count`, and `truncated` fields describe the returned selection accurately; additive `source_total_count`, `returned_count`, `limit_per_query`, `total_limit`, `first_per_file`, and `sample` fields make the applied contract auditable. Replay commands are serialized from normalized parsed options, use POSIX-safe single-quote escaping, and retain raw/exact/prefix modes, path/language/facet/guard filters, selection controls, evidence formatting, duplicate preflight, and issue hints. | | Recipe SARIF | `search --recipe --format sarif` emits one result per bounded recipe result. Rule IDs use `recipe/query`; standard `fingerprints.cdidx/v1` values are derived from the normalized source location; result properties preserve recipe/query identity, severity, confidence, and per-query truncation; run properties preserve scope, applied result limits, aggregate counts, and conservative omitted-result metadata. Bound SARIF with `--limit` / `--total-limit`; row selectors such as `--sample`, `--first-per-file`, and `--per-file-limit` are rejected instead of being silently ignored. Recipe severity maps `critical` / `high` to `error`, `medium` to `warning`, and `low` / `info` to `note`. | | Recipe classifier output | Recipe run JSON may add `audit_classifications` to individual `CompactSearchResult` rows when a recipe classifier can classify the hit, and query/count payloads may add `classifier_counts` when classified rows are present. These fields are additive; use them to separate triage domains such as DTO/result-wrapper `.Result` properties versus Task/ValueTask blocking waits without changing the raw search query. | @@ -4744,7 +4744,7 @@ help はすべてこのレジストリを参照します。field 名は大文字 | 高ボリューム応答の bounded 契約 | `search`、`definition`、`find`、`status`、`hotspots`、`references`、`callers`、`callees`、`symbols`、`files`、`languages`、`impact`、`map` は、それぞれの schema が公開する共通 bounded-response control に対応します。新しく発行する opaque な `--cursor ` は offset を command / query / filter と index generation に束縛し、移行用に legacy の `response:v1::` も受理します。選択条件または generation を変えて再利用すると restart-required の案内付きで失敗します。`search --format compact`、`symbols --format compact`、`files --format compact` は bounded 契約を自動選択し、`search --json=array --json-envelope` は opt-in の array envelope、`languages --json` は paging または `--max-json-bytes` 指定時に同じ契約を使います。既存 compact の root と location row は維持したまま共通 metadata を追加します。metadata は `returned_count`、取得可能な場合は authoritative な `total_count`、`omitted_count`、`remaining_count`、`cursor_offset`、`page_limit`、`has_more`、`next_cursor`、`result_stable_at`、`pagination_window_limit`、`pagination_window_exhausted` を返します。safety window は 10,000 row で、上限到達時は次の request が拒否する cursor を返さず `next_cursor` を抑止します。pageable command は `offset + limit` 件を serialize せず、cursor offset を database / scan layer へ渡します。`find --all` の partial scan cursor は次の path / line を保持し、再利用時は最後に scan した line の次から継続します。`hotspots` と `impact` は active な主要 nested collection を `results` としてページングし、`metadata.primary_collection` でその名前を示し、scalar / container evidence は `metadata.response_context` に保持します。`callers.path,callers.depth` のような dotted field で collection と row field を同時に選べます。`--max-json-bytes` は最後の改行を含み、完全な envelope が収まるまで末尾の完全な row を省略します。`definition` は既定で metadata-only のままで、明示的な `--body` は `body`、`body_content`、`all` で保持し、それ以外の projection では materialize 前に抑止します。`map --sections` は section-level projection として残り、dotted な bounded field は選択した array section を section 固有の総件数付きでページングし、scalar projection は不要な ranking array を構築しません。 | | bounded 応答の edge case | `impact` は選択された nested collection だけに cursor offset を適用するため、definition page の重複や caller / fallback mode の変化を防ぎます。通常の `map --compact` は既存の section array と truncation payload を維持し、collection projection が `--summary-only` または除外する `--sections` filter で失われる組み合わせは拒否します。明示的な definition body field は compact default より優先します。profile / verbose record は `metadata.stream_control_records` へ移し、parser / capture failure の error envelope は active な hard byte cap に収まる場合だけ出力します。 | | `--count --json` envelope | `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`find`、`impact`、`unused` の count-only JSON は単一の自動化向け object です。常に `count`、適用済み `query_context`、freshness metadata（`indexed_file_count`、`indexed_at`、`freshness_available`）、trust flag の `degraded` / `authoritative_count` を含みます。matched-file total を持つ command は `files` と古い互換 alias の `file_count` も含みます。`file_count` は `files` と同じ値を持つ互換用 field として残り、少なくとも次の major release までは削除予定はありません。`unused --count --json` は `returned_bucket_counts`、`returned_contract_domain_counts`、`summary.by_bucket` / `summary.by_confidence` / `summary.by_contract_domain` も含みます。`authoritative_count=false` は readiness または graph/exact trust signal により count が authoritative ではないことを示し、freshness field は count に使った index snapshot を説明します。 | -| recipe row selection | row を返す recipe path は `ApplySearchOutputSelection` を共有し、`--first-per-file` と固定 seed の決定的な `--sample` を、有効な query ごとの limit / 残り total limit より先に適用します。sample 用 fetch envelope は少なくとも要求 sample 数を基準に sizing します。aggregate / compact の query DTO、run summary、issue-draft の source DTO、NDJSON terminal は `source_total`、`selected_total`、`returned`、`selector_omitted_count`、`limit_omitted_count` を公開し、`source_total_authoritative` / `source_total_lower_bound` で完全な population と bounded な観測を区別します。適用順の `selectors` entry は各段階の input / output / omission count と sample の size / mode / seed を保持し、nullable な `selection_reason` / `selection_omitted_count` は互換用 summary として維持します。search の `query_context.row_selectors` は適用済み selector 設定を記録します。selection だけによる省略は matched / omitted の lower bound を更新しますが、`truncated`、`has_more`、`next_cursor` は設定しません。後続の limit が選択済み row を truncate する場合、limit truncation は表示しますが raw database cursor は selector state を保持できないため `next_cursor` を抑止し、同じ理由で selector と受け取った `--cursor` の併用も拒否します。compact / issue-draft の生成 replay command は selector を保持します。count、aggregation、summary-only compact の shape は `--first-per-file` / `--sample` を拒否し、すべての recipe shape は grouped 専用の `--per-file-limit` を拒否します。 | +| search row selection | row を返す plain search / recipe path は `ApplySearchOutputSelection` を共有し、`--first-per-file` と固定 seed の決定的な `--sample` を、有効な query ごとの limit / 残り total limit より先に適用します。sample 用 fetch envelope は少なくとも要求 sample 数を基準に sizing します。aggregate / compact の query DTO、plain compact root、run summary、issue-draft の source DTO、NDJSON terminal、bounded array envelope の stream terminal は `source_total`、`selected_total`、`returned`、`selector_omitted_count`、`limit_omitted_count` を公開し、`source_total_authoritative` / `source_total_lower_bound` で完全な population と bounded な観測を区別します。guard filter、origin / facet の後段 filter、candidate window の枯渇、recipe の file-reject 後段 filter は lower-bound authority にします。適用順の `selectors` entry は各段階の input / output / omission count と sample の size / mode / seed を保持し、nullable な `selection_reason` / `selection_omitted_count` は互換用 summary として維持します。bounded plain-search selection は一度だけ計算し、その selected page を compact / envelope serialize で再利用します。search の `query_context.row_selectors` は適用済み selector 設定を記録します。selection だけによる省略は matched / omitted の lower bound を更新しますが、`truncated`、`has_more`、`next_cursor` は設定しません。後続の limit が選択済み row を truncate する場合、limit truncation は表示しますが raw database cursor は selector state を保持できないため `next_cursor` を抑止し、同じ理由で selector と受け取った `--cursor` の併用も拒否します。compact / issue-draft の生成 replay command は selector を保持します。count、aggregation、named-query、recipe-list、results-only、metadata を持たない array、非対応 formatted、summary-only compact の shape は `--first-per-file` / `--sample` を拒否し、すべての recipe shape は grouped 専用の `--per-file-limit` を拒否します。 | | ad-hoc issue-draft selection | `search --format issue-drafts` は filter 済みの ad-hoc 母集団全体を読み、`--first-per-file`、決定的な `--sample`、`min(--limit, --total-limit)` の順に適用します。guard 付き検索は有限の candidate inspection 契約を維持し、`source_total_count` を省略し、観測下限を `source_minimum_count`、非 authoritative 状態を `source_total_count_authoritative=false`、bounded fetch を `source_fetch_limit` で報告し、母集団が未完了であることを `truncated=true` で保持します。既存の `result_count`、`result_limit`、`omitted_count`、`truncated` field は返却 selection を正確に表し、additive な `source_total_count`、`returned_count`、`limit_per_query`、`total_limit`、`first_per_file`、`sample` field により適用済み契約を監査できます。replay command は正規化済み parse option から serialize し、POSIX-safe な単一引用符 escape を使い、raw / exact / prefix mode、path / language / facet / guard filter、selection control、evidence formatting、duplicate preflight、issue hint を維持します。 | | Recipe SARIF | `search --recipe --format sarif` は、上限付き recipe result ごとに result を1件出力します。rule ID は `recipe/query` を使い、標準の `fingerprints.cdidx/v1` は正規化済み source location から導出します。result properties は recipe/query identity、severity、confidence、query ごとの truncation を保持し、run properties は scope、適用済み result limit、集計 count、保守的な omitted-result metadata を保持します。SARIF の上限には `--limit` / `--total-limit` を使い、`--sample`、`--first-per-file`、`--per-file-limit` のような row selector は黙って無視せず拒否します。recipe severity は `critical` / `high` を `error`、`medium` を `warning`、`low` / `info` を `note` に対応付けます。 | | Recipe classifier output | recipe classifier が hit を分類できる場合、recipe run JSON は個別の `CompactSearchResult` row に `audit_classifications` を追加することがあり、分類済み row がある query / count payload は `classifier_counts` を追加することがあります。これらは additive field です。raw search query を変えずに、DTO / result-wrapper の `.Result` property と Task / ValueTask の blocking wait などの triage domain を分離するために使います。 | diff --git a/USER_GUIDE.md b/USER_GUIDE.md index d7168739d3..fcc444b71d 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1382,17 +1382,21 @@ names, aggregate counts, per-query counts, and query freshness. Recipe count aggregations support `--count-by path|file|symbol|origin|return-type|subsystem`, `--group-by file|symbol|origin|return-type|subsystem --count`, and `--unique path|file|symbol|origin|return-type|subsystem`. -Row-producing recipe modes (text, aggregate JSON, compact JSON, NDJSON, and -issue drafts) apply `--first-per-file` and fixed-seed deterministic -`--sample ` before +Row-producing search and recipe modes (text, aggregate JSON, compact JSON, +NDJSON, JSON array envelopes, and issue drafts) apply `--first-per-file` and +fixed-seed deterministic `--sample ` before the effective per-query `--limit` / cross-query `--total-limit`. Aggregate JSON -and compact query objects, run summaries, issue-draft `source` objects, and -NDJSON terminal records distinguish `source_total`, `selected_total`, -`returned`, `selector_omitted_count`, and `limit_omitted_count`. Their +and compact query objects, plain compact roots, run summaries, issue-draft +`source` objects, NDJSON terminal records, and array-envelope +`metadata.stream_terminal` objects distinguish `source_total`, +`selected_total`, `returned`, `selector_omitted_count`, and +`limit_omitted_count`. Their `selectors` array records each applied selector in execution order, including per-stage input/output/omission counts and the sample size, mode, and seed. `source_total_authoritative` says whether the bounded fetch observed the whole -source population; otherwise `source_total_lower_bound` is also present. The +source population; guard filters, origin/facet post-filters, bounded candidate +windows, and recipe file-reject post-filters conservatively produce +`source_total_authoritative=false` with `source_total_lower_bound`. The older `selection_reason` and `selection_omitted_count` fields remain as compatibility summaries. Search `query_context.row_selectors` exposes every applied selector with the same sample mode and seed. When a hard @@ -1407,8 +1411,12 @@ rerun instead. For the same reason, recipe row selectors reject an incoming `--cursor`. Generated compact and issue-draft replay commands retain the active selector. Count, aggregation, and summary-only compact recipe output reject row-selection controls because -they cannot represent selected rows, and recipe execution rejects -`--per-file-limit` because it does not produce grouped search output. +they cannot represent selected rows. Plain count/aggregation, named-query and +recipe-list modes, `--results-only`, metadata-free `--json=array`, and formatted +row outputs without selector accounting also reject them instead of silently +ignoring them. Add `--json-envelope` to an array request to retain selector +accounting. Recipe execution rejects `--per-file-limit` because it does not +produce grouped search output. Recipe SARIF emits one bounded finding per returned recipe result. Its rule IDs use `recipe/query`, result fingerprints are stable for the recipe/query/source location, and result/run properties preserve severity, confidence, scope, @@ -4516,15 +4524,19 @@ child query 全体の emitted row 数を制限でき、NDJSON では `--max-json recipe count output は `--format count --summary-only --max-json-bytes ` により、recipe / scope 名、 aggregate count、query ごとの count、query freshness だけを出力できます。recipe の count aggregation は `--count-by path|file|symbol|origin|return-type|subsystem`、 `--group-by file|symbol|origin|return-type|subsystem --count`、`--unique path|file|symbol|origin|return-type|subsystem` に対応します。 -row を返す recipe mode（text、aggregate JSON、compact JSON、NDJSON、issue draft）は、 -`--first-per-file` と固定 seed の決定的な `--sample ` を、有効な query ごとの `--limit` / +row を返す search / recipe mode（text、aggregate JSON、compact JSON、NDJSON、 +JSON array envelope、issue draft）は、`--first-per-file` と固定 seed の決定的な +`--sample ` を、有効な query ごとの `--limit` / query 全体の `--total-limit` より先に適用します。aggregate JSON / compact の query object は -run summary、issue-draft の `source` object、NDJSON terminal record と同様に、 -`source_total`、`selected_total`、`returned`、`selector_omitted_count`、 -`limit_omitted_count` を分けて返します。`selectors` array は適用順の各 selector について、 +plain compact の root、run summary、issue-draft の `source` object、NDJSON terminal record、 +array envelope の `metadata.stream_terminal` と同様に、`source_total`、`selected_total`、 +`returned`、`selector_omitted_count`、`limit_omitted_count` を分けて返します。 +`selectors` array は適用順の各 selector について、 各段階の入力件数、出力件数、省略件数、および sample の size / mode / seed を記録します。 bounded fetch が source population 全体を観測できたかは `source_total_authoritative` で示し、 -そうでない場合は `source_total_lower_bound` も返します。従来の `selection_reason` と +guard filter、origin / facet の後段 filter、bounded candidate window、recipe の file-reject +後段 filter がある場合は保守的に `source_total_authoritative=false` と +`source_total_lower_bound` を返します。従来の `selection_reason` と `selection_omitted_count` は互換用 summary として維持します。 search の `query_context.row_selectors` も適用済み selector と同じ sample mode / seed を公開します。 hard な `--max-json-bytes` cap で NDJSON terminal に追加 accounting field が収まらない場合、 @@ -4535,8 +4547,12 @@ selection だけによる省略は matched / omitted count に含まれますが `next_cursor` は抑止します。この場合は該当 limit を増やして再実行してください。同じ理由で、 recipe の row selector は受け取った `--cursor` も拒否します。compact / issue-draft が生成する replay command は有効な selector を保持します。count、aggregation、summary-only compact の -recipe output は選択済み row を表現できないため row-selection control を拒否し、recipe -execution は grouped search output を生成しないため `--per-file-limit` を拒否します。 +recipe output は選択済み row を表現できないため row-selection control を拒否します。 +plain count / aggregation、named-query、recipe-list、`--results-only`、metadata を持たない +`--json=array`、selector accounting を持たない formatted row output も、黙って無視せず +row-selection control を拒否します。array request で accounting を保持するには +`--json-envelope` を追加します。recipe execution は grouped search output を生成しないため +`--per-file-limit` を拒否します。 recipe SARIF は返却された recipe result ごとに上限付き finding を1件出力します。rule ID は `recipe/query` を使い、result fingerprint は recipe / query / source location に対して安定し、 result / run properties は severity、confidence、scope、適用済み result limit、 diff --git a/changelog.d/unreleased/4843.fixed.md b/changelog.d/unreleased/4843.fixed.md index eb25fcdf08..c32bb262ea 100644 --- a/changelog.d/unreleased/4843.fixed.md +++ b/changelog.d/unreleased/4843.fixed.md @@ -18,8 +18,8 @@ affected: ## English -- **Sampling and first-per-file output now reports the source, selected, and returned populations separately (#4843)** — recipe JSON, compact output, issue drafts, and NDJSON terminals now expose per-selector accounting, limit omissions, deterministic sample mode/seed, and bounded-source authority. Search query context lists every applied selector, while count and other non-row recipe modes reject selectors with a stable usage error. +- **Sampling and first-per-file output now reports the source, selected, and returned populations separately (#4843)** — recipe JSON, plain compact output, array envelopes, issue drafts, and NDJSON terminals now expose per-selector accounting, limit omissions, deterministic sample mode/seed, and bounded-source authority. Search query context lists every applied selector; guard and post-filtered populations are reported as lower bounds; count, aggregation, named-query, recipe-list, results-only, and metadata-free array modes reject selectors with a stable usage error instead of silently ignoring them. ## 日本語 -- **sample / first-per-file 出力で選択前・選択後・返却後の件数を分けて報告するようになりました (#4843)** — recipe JSON、compact 出力、issue draft、NDJSON terminal は selector ごとの件数、limit による省略、決定的 sample の mode / seed、bounded な source 件数の authority を公開します。search の query context は適用済み selector をすべて列挙し、count など row を返さない recipe mode は selector を安定した usage error で拒否します。 +- **sample / first-per-file 出力で選択前・選択後・返却後の件数を分けて報告するようになりました (#4843)** — recipe JSON、plain compact 出力、array envelope、issue draft、NDJSON terminal は selector ごとの件数、limit による省略、決定的 sample の mode / seed、bounded な source 件数の authority を公開します。search の query context は適用済み selector をすべて列挙し、guard / 後段 filter 付きの population は lower bound として報告します。count、aggregation、named-query、recipe-list、results-only、metadata を持たない array mode は selector を黙って無視せず、安定した usage error で拒否します。 diff --git a/src/CodeIndex/Cli/QueryCommandRunner.Search.cs b/src/CodeIndex/Cli/QueryCommandRunner.Search.cs index 5dcafae0e6..7e6b029345 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.Search.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.Search.cs @@ -268,6 +268,14 @@ private static int RunSearchCore( return CommandExitCodes.UsageError; if (options.ListRecipes) { + if (options.FirstPerFile || options.SampleSize.HasValue) + { + WriteUsageError( + "row-selection controls are not supported with --list-recipes because recipe discovery does not emit search rows.", + GetUsageLineOrThrow(usageCommandName), + "Remove --first-per-file / --sample, or execute a recipe or plain search that returns rows."); + return CommandExitCodes.UsageError; + } if (options.RecipeName != null || options.NamedSearchQueries.Count > 0 || options.ExtraNames.Count > 0) { WriteUsageError( @@ -297,6 +305,14 @@ private static int RunSearchCore( } if (options.NamedSearchQueries.Count > 0) { + if (options.FirstPerFile || options.SampleSize.HasValue) + { + WriteUsageError( + "row-selection controls are not supported with --named-query because named batches do not expose selector accounting.", + GetUsageLineOrThrow("search"), + "Remove --first-per-file / --sample, or run each query as a plain search or recipe row output."); + return CommandExitCodes.UsageError; + } if (options.Query != null || options.RecipeName != null || options.ExtraNames.Count > 0) { WriteUsageError( @@ -426,10 +442,11 @@ private static int RunSearchCore( || options.GroupBy != null || options.CountBy != null || options.UniqueBy != null + || options.ResultsOnly || (options.SummaryOnly && (options.Compact || options.OutputFormat == OutputFormatCompact)))) { WriteUsageError( - "recipe row-selection controls cannot be combined with count, aggregation, or summary-only compact output.", + "recipe row-selection controls cannot be combined with count, aggregation, results-only, or summary-only compact output.", GetUsageLineOrThrow("search"), "Remove --first-per-file / --sample to keep the non-row output, or choose text, JSON, compact, NDJSON, or issue-drafts row output."); return CommandExitCodes.UsageError; @@ -550,6 +567,48 @@ private static int RunSearchCore( } if (TryWriteUnexpectedExtraPositionals("search", options)) return CommandExitCodes.UsageError; + if ((options.FirstPerFile || options.SampleSize.HasValue) + && (options.CountOnly + || options.GroupBy != null + || options.CountBy != null + || options.UniqueBy != null + || options.OutputFormat == OutputFormatGrouped)) + { + WriteUsageError( + "search row-selection controls cannot be combined with count or aggregation output.", + GetUsageLineOrThrow("search"), + "Remove --first-per-file / --sample to count the full filtered population, or choose a row output that reports selector accounting."); + return CommandExitCodes.UsageError; + } + if ((options.FirstPerFile || options.SampleSize.HasValue) && options.ResultsOnly) + { + WriteUsageError( + "search row-selection controls cannot be combined with --results-only because that stream omits selector accounting.", + GetUsageLineOrThrow("search"), + "Remove --results-only to retain the NDJSON terminal record, or remove --first-per-file / --sample."); + return CommandExitCodes.UsageError; + } + if ((options.FirstPerFile || options.SampleSize.HasValue) + && options.JsonOutputFormat == JsonOutputFormatArray) + { + WriteUsageError( + "search row-selection controls cannot be combined with metadata-free --json=array output.", + GetUsageLineOrThrow("search"), + "Add --json-envelope to retain selector accounting, use --json=ndjson / --format compact, or remove --first-per-file / --sample."); + return CommandExitCodes.UsageError; + } + if ((options.FirstPerFile || options.SampleSize.HasValue) + && options.OutputFormat is not OutputFormatText + and not OutputFormatJson + and not OutputFormatCompact + and not OutputFormatIssueDrafts) + { + WriteUsageError( + "search row-selection controls are only supported by text, JSON, compact, and issue-drafts row output.", + GetUsageLineOrThrow("search"), + "Choose an output shape that reports selector accounting, or remove --first-per-file / --sample."); + return CommandExitCodes.UsageError; + } if (options.GroupBy != null) { if (!IsSupportedSearchGroupByValue(options.GroupBy)) @@ -724,8 +783,8 @@ private static int RunSearchCore( var groupedCounts = options.OutputFormat == OutputFormatGrouped ? CountSearchMatches(reader, options, exactSearch) : default; - var displayRows = ReadSearchDisplayRows(reader, options, exactSearch); - var selection = ApplySearchOutputSelection(displayRows, options); + var displayRows = ReadSearchDisplayRows(reader, options, exactSearch, out var boundedSelection); + var selection = boundedSelection ?? ApplySearchOutputSelection(displayRows, options); displayRows = selection.Rows; if (displayRows.Count == 0) { @@ -739,6 +798,13 @@ private static int RunSearchCore( var groupedExitCode = WriteGroupedSearchResults([], groupedCounts, options, jsonOptions); return groupedExitCode == CommandExitCodes.Success ? ZeroResultExitCode(options) : groupedExitCode; } + if (options.Json + && options.OutputFormat == OutputFormatCompact + && selection.Selectors.Count > 0) + { + var compactExitCode = WriteCompactSearchResults([], options, jsonOptions, selection); + return compactExitCode == CommandExitCodes.Success ? ZeroResultExitCode(options) : compactExitCode; + } if (options.Json && TryWriteEmptySearchJsonWithOptionalByteLimit(options, jsonOptions, out var emptyJsonExitCode)) return emptyJsonExitCode == CommandExitCodes.Success ? ZeroResultExitCode(options) : emptyJsonExitCode; if (options.Json && TryWriteEmptyFormattedResult(options, jsonOptions)) @@ -836,7 +902,7 @@ selection.TruncationReason is "sample" or "first_per_file" } if (options.OutputFormat == OutputFormatCompact) { - return WriteCompactSearchResults(compactResults, options, jsonOptions); + return WriteCompactSearchResults(compactResults, options, jsonOptions, selection); } if (options.OutputFormat == OutputFormatGrouped) { diff --git a/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs b/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs index d4049aadc9..9bc65c1b41 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs @@ -1320,7 +1320,8 @@ private static int RunSearchIssueDrafts( return WithDb(options, jsonOptions, reader => { var resultLimit = GetAdHocIssueDraftResultLimit(options); - var sourceTotalCountAuthoritative = options.GuardFilters.Count == 0; + var sourceTotalCountAuthoritative = options.GuardFilters.Count == 0 + && !HasSearchOriginFilters(options); var sourceFetchLimit = sourceTotalCountAuthoritative ? int.MaxValue : GetSearchRecipeFetchLimit(options, resultLimit); @@ -1461,7 +1462,12 @@ private static List CollectSearchRecipeQueryR guardScope: options.GuardScope, requiredPathPatterns: GetSearchRecipeRequiredPathPatterns(options, recipeQuery), resultRanking: GetSearchRecipeResultRanking(recipeQuery.ResultRanking, resultLimit)); - var sourceTotalAuthoritative = results.Count < fetchLimit; + var sourceTotalAuthoritative = IsSearchRecipeSourceTotalAuthoritative( + options, + recipeQuery, + guardFilters, + results.Count, + fetchLimit); results = ApplySearchRecipeFileRejectQueries(reader, results, options, recipeQuery); var rows = BuildSearchDisplayRows(results, options, exact, recipeQuery.Query, rawFtsOverride: false, recipeQuery: recipeQuery); var outputSelection = ApplySearchOutputSelection(rows, options, resultLimit, sourceTotalAuthoritative); @@ -1557,7 +1563,12 @@ private static List CollectSearchRecip guardScope: options.GuardScope, requiredPathPatterns: GetSearchRecipeRequiredPathPatterns(options, recipeQuery), resultRanking: GetSearchRecipeResultRanking(recipeQuery.ResultRanking, resultLimit)); - var sourceTotalAuthoritative = results.Count < fetchLimit; + var sourceTotalAuthoritative = IsSearchRecipeSourceTotalAuthoritative( + options, + recipeQuery, + guardFilters, + results.Count, + fetchLimit); results = ApplySearchRecipeFileRejectQueries(reader, results, options, recipeQuery); var rows = BuildSearchDisplayRows(results, options, exact, recipeQuery.Query, recipeQuery: recipeQuery); var outputSelection = ApplySearchOutputSelection(rows, options, resultLimit, sourceTotalAuthoritative); @@ -1627,6 +1638,17 @@ private static List CollectSearchRecip ? selection.TruncationReason : null; + private static bool IsSearchRecipeSourceTotalAuthoritative( + QueryCommandOptions options, + SearchAuditRecipeQuery recipeQuery, + IReadOnlyCollection guardFilters, + int resultCount, + int fetchLimit) + => guardFilters.Count == 0 + && recipeQuery.RejectFileQueries.Count == 0 + && !HasSearchOriginFilters(BuildSearchDisplayFacetFilters(options, recipeQuery)) + && resultCount < fetchLimit; + private static int GetSearchRecipeFetchLimit(QueryCommandOptions options, int resultLimit) { var selectionTarget = resultLimit > 0 && options.SampleSize.HasValue diff --git a/src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs b/src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs index d447f9748f..0f88e43e2f 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.SearchResults.cs @@ -373,8 +373,10 @@ private static SearchOutputSelection ApplySearchOutputSelection(List rows, @@ -1160,8 +1162,13 @@ private static List CombineExclusiveSearchFilters(IReadOnlyList .FirstOrDefault(); } - private static List ReadSearchDisplayRows(DbReader reader, QueryCommandOptions options, bool exact) + private static List ReadSearchDisplayRows( + DbReader reader, + QueryCommandOptions options, + bool exact, + out SearchOutputSelection? boundedSelection) { + boundedSelection = null; var responseOffset = JsonEnvelopeWrapper.GetBoundedResponseOffset("search"); var boundedPageLimit = JsonEnvelopeWrapper.GetBoundedResponseLimit("search"); if (boundedPageLimit.HasValue @@ -1189,21 +1196,34 @@ private static List ReadSearchDisplayRows(DbReader reader, Que SearchOriginFilterMaxCandidates); } - var rawWindowComplete = boundedRows.Count < SearchOriginFilterMaxCandidates; - var selectedRows = ApplySearchPostSelectors( + var sourceTotalAuthoritative = boundedRows.Count < SearchOriginFilterMaxCandidates + && options.GuardFilters.Count == 0 + && !HasSearchOriginFilters(options); + var fullSelection = ApplySearchOutputSelection( boundedRows, options, - []); - JsonEnvelopeWrapper.ReportBoundedResponseTotal( - "search", - selectedRows.Count, - rawWindowComplete); - return selectedRows + int.MaxValue, + sourceTotalAuthoritative); + var pageRows = fullSelection.Rows .Skip(responseOffset) - .Take(boundedPageLimit.Value == int.MaxValue - ? int.MaxValue - : boundedPageLimit.Value + 1) + .Take(boundedPageLimit.Value) .ToList(); + var limitOmittedCount = Math.Max(0, fullSelection.SelectedTotal - pageRows.Count); + var limitTruncated = limitOmittedCount > 0; + boundedSelection = fullSelection with + { + Rows = pageRows, + Returned = pageRows.Count, + LimitOmittedCount = limitOmittedCount, + Truncated = pageRows.Count < fullSelection.SourceTotal, + LimitTruncated = limitTruncated, + TruncationReason = fullSelection.TruncationReason ?? (limitTruncated ? "limit" : null), + }; + JsonEnvelopeWrapper.ReportBoundedResponseTotal( + "search", + fullSelection.SelectedTotal, + sourceTotalAuthoritative); + return pageRows; } var requestedLimit = GetSearchDisplayCandidateLimit(options); @@ -1701,10 +1721,14 @@ private static string BuildJsonStreamDoneLine( private static JsonSerializerOptions GetCompactJsonOptions(JsonSerializerOptions jsonOptions) => jsonOptions.WriteIndented ? new JsonSerializerOptions(jsonOptions) { WriteIndented = false } : jsonOptions; - private static int WriteCompactSearchResults(IEnumerable results, QueryCommandOptions options, JsonSerializerOptions jsonOptions) + private static int WriteCompactSearchResults( + IEnumerable results, + QueryCommandOptions options, + JsonSerializerOptions jsonOptions, + SearchOutputSelection? selection = null) { using var writer = new StringWriter(CultureInfo.InvariantCulture); - WriteCompactSearchResults(writer, results, options, jsonOptions); + WriteCompactSearchResults(writer, results, options, jsonOptions, selection); return WriteJsonObjectWithOptionalByteLimit( writer.ToString().TrimEnd('\r', '\n'), options, @@ -1712,12 +1736,53 @@ private static int WriteCompactSearchResults(IEnumerable re "Reduce --limit, --snippet-lines, or use `--json=ndjson --max-json-bytes` for streaming output."); } - private static void WriteCompactSearchResults(TextWriter writer, IEnumerable results, QueryCommandOptions options, JsonSerializerOptions jsonOptions) + private static void WriteCompactSearchResults( + TextWriter writer, + IEnumerable results, + QueryCommandOptions options, + JsonSerializerOptions jsonOptions, + SearchOutputSelection? selection = null) { var locations = results.Select(result => new FormattedLocation( result.Path, result.MatchLines.Count > 0 ? result.MatchLines[0] : result.ChunkStartLine)); - writer.WriteLine(BuildCompactLocationsPayload(locations, options, jsonOptions).ToJsonString(jsonOptions)); + var payload = BuildCompactLocationsPayload(locations, options, jsonOptions); + if (selection is { Selectors.Count: > 0 }) + AddSearchSelectionAccounting(payload, selection); + writer.WriteLine(payload.ToJsonString(jsonOptions)); + } + + private static void AddSearchSelectionAccounting(JsonObject payload, SearchOutputSelection selection) + { + payload["source_total"] = selection.SourceTotal; + payload["source_total_authoritative"] = selection.SourceTotalAuthoritative; + payload["source_total_lower_bound"] = selection.SourceTotalAuthoritative + ? null + : selection.SourceTotal; + payload["selected_total"] = selection.SelectedTotal; + payload["returned"] = selection.Returned; + payload["selector_omitted_count"] = selection.SelectorOmittedCount; + payload["limit_omitted_count"] = selection.LimitOmittedCount; + var selectors = new JsonArray(); + foreach (var selector in selection.Selectors) + { + var selectorPayload = new JsonObject + { + ["mode"] = selector.Mode, + ["applied"] = selector.Applied, + ["input_total"] = selector.InputTotal, + ["output_total"] = selector.OutputTotal, + ["omitted_count"] = selector.OmittedCount, + }; + if (selector.SampleSize.HasValue) + selectorPayload["sample_size"] = selector.SampleSize.Value; + if (selector.SampleMode is not null) + selectorPayload["sample_mode"] = selector.SampleMode; + if (selector.Seed.HasValue) + selectorPayload["seed"] = selector.Seed.Value; + selectors.Add(selectorPayload); + } + payload["selectors"] = selectors; } private static void WriteJsonArray(IEnumerable items, Action writeItem, JsonSerializerOptions jsonOptions) diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs index d43c140bec..a251bc97d1 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs @@ -7210,7 +7210,8 @@ public void RunSearch_RecipeRowSelectionAppliesToJsonCompactAndNdjson_Issue4759( Assert.Equal(4, sampleSummary.GetProperty("selector_omitted_count").GetInt32()); Assert.Equal(0, sampleSummary.GetProperty("limit_omitted_count").GetInt32()); Assert.False(sampleSummary.GetProperty("cursoring_available").GetBoolean()); - Assert.True(sampleSummary.GetProperty("source_total_authoritative").GetBoolean()); + Assert.False(sampleSummary.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(5, sampleSummary.GetProperty("source_total_lower_bound").GetInt32()); var sampleQuery = Assert.Single(sampleDocument.RootElement.GetProperty("queries").EnumerateArray()); Assert.Equal(1, sampleQuery.GetProperty("count").GetInt32()); Assert.Equal(1, sampleQuery.GetProperty("emitted_count").GetInt32()); @@ -7219,7 +7220,8 @@ public void RunSearch_RecipeRowSelectionAppliesToJsonCompactAndNdjson_Issue4759( Assert.Equal("sample", sampleQuery.GetProperty("selection_reason").GetString()); Assert.Equal(4, sampleQuery.GetProperty("selection_omitted_count").GetInt32()); Assert.Equal(5, sampleQuery.GetProperty("source_total").GetInt32()); - Assert.True(sampleQuery.GetProperty("source_total_authoritative").GetBoolean()); + Assert.False(sampleQuery.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(5, sampleQuery.GetProperty("source_total_lower_bound").GetInt32()); Assert.Equal(1, sampleQuery.GetProperty("selected_total").GetInt32()); Assert.Equal(1, sampleQuery.GetProperty("returned").GetInt32()); Assert.Equal(4, sampleQuery.GetProperty("selector_omitted_count").GetInt32()); @@ -7272,7 +7274,8 @@ public void RunSearch_RecipeRowSelectionAppliesToJsonCompactAndNdjson_Issue4759( Assert.Equal("sample", compactQuery.GetProperty("selection_reason").GetString()); Assert.Equal(4, compactQuery.GetProperty("selection_omitted_count").GetInt32()); Assert.Equal(5, compactQuery.GetProperty("source_total").GetInt32()); - Assert.True(compactQuery.GetProperty("source_total_authoritative").GetBoolean()); + Assert.False(compactQuery.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(5, compactQuery.GetProperty("source_total_lower_bound").GetInt32()); Assert.Equal(1, compactQuery.GetProperty("selected_total").GetInt32()); Assert.Equal(1, compactQuery.GetProperty("returned").GetInt32()); Assert.Equal(4, compactQuery.GetProperty("selector_omitted_count").GetInt32()); @@ -7315,7 +7318,8 @@ public void RunSearch_RecipeRowSelectionAppliesToJsonCompactAndNdjson_Issue4759( Assert.Equal("sample", draft.GetProperty("source").GetProperty("selection_reason").GetString()); Assert.Equal(4, draft.GetProperty("source").GetProperty("selection_omitted_count").GetInt32()); Assert.Equal(5, draft.GetProperty("source").GetProperty("source_total").GetInt32()); - Assert.True(draft.GetProperty("source").GetProperty("source_total_authoritative").GetBoolean()); + Assert.False(draft.GetProperty("source").GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(5, draft.GetProperty("source").GetProperty("source_total_lower_bound").GetInt32()); Assert.Equal(1, draft.GetProperty("source").GetProperty("selected_total").GetInt32()); Assert.Equal(1, draft.GetProperty("source").GetProperty("returned").GetInt32()); Assert.Equal(4, draft.GetProperty("source").GetProperty("selector_omitted_count").GetInt32()); @@ -7338,7 +7342,8 @@ public void RunSearch_RecipeRowSelectionAppliesToJsonCompactAndNdjson_Issue4759( Assert.Equal("sample", ndjsonTerminal.RootElement.GetProperty("selection_reason").GetString()); Assert.Equal(4, ndjsonTerminal.RootElement.GetProperty("selection_omitted_count").GetInt32()); Assert.Equal(5, ndjsonTerminal.RootElement.GetProperty("source_total").GetInt32()); - Assert.True(ndjsonTerminal.RootElement.GetProperty("source_total_authoritative").GetBoolean()); + Assert.False(ndjsonTerminal.RootElement.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(5, ndjsonTerminal.RootElement.GetProperty("source_total_lower_bound").GetInt32()); Assert.Equal(1, ndjsonTerminal.RootElement.GetProperty("selected_total").GetInt32()); Assert.Equal(1, ndjsonTerminal.RootElement.GetProperty("returned").GetInt32()); Assert.Equal(4, ndjsonTerminal.RootElement.GetProperty("selector_omitted_count").GetInt32()); @@ -7399,7 +7404,8 @@ public void RunSearch_RecipeSelectionAccountingReportsSourceSelectorsAndLimit_Is var repeatQuery = Assert.Single(repeatDocument.RootElement.GetProperty("queries").EnumerateArray()); Assert.Equal(126, sampleQuery.GetProperty("source_total").GetInt32()); - Assert.True(sampleQuery.GetProperty("source_total_authoritative").GetBoolean()); + Assert.False(sampleQuery.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(126, sampleQuery.GetProperty("source_total_lower_bound").GetInt32()); Assert.Equal(5, sampleQuery.GetProperty("selected_total").GetInt32()); Assert.Equal(5, sampleQuery.GetProperty("returned").GetInt32()); Assert.Equal(121, sampleQuery.GetProperty("selector_omitted_count").GetInt32()); @@ -7412,6 +7418,83 @@ public void RunSearch_RecipeSelectionAccountingReportsSourceSelectorsAndLimit_Is sampleQuery.GetProperty("results").GetRawText(), repeatQuery.GetProperty("results").GetRawText()); + var (compactExitCode, compactStdout, compactStderr) = CaptureConsole(() => + ProgramRunner.Run( + [ + "search", + "Console.WriteLine", + "--db", dbPath, + "--exact-substring", + "--format", "compact", + "--sample", "5", + "--limit", "20", + ], + _jsonOptions, + "test")); + + Assert.Equal(CommandExitCodes.Success, compactExitCode); + Assert.Equal(string.Empty, compactStderr); + using var compactDocument = ParseJsonOutput(compactStdout); + Assert.Equal(126, compactDocument.RootElement.GetProperty("source_total").GetInt32()); + Assert.True(compactDocument.RootElement.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(5, compactDocument.RootElement.GetProperty("selected_total").GetInt32()); + Assert.Equal(5, compactDocument.RootElement.GetProperty("returned").GetInt32()); + Assert.Equal(121, compactDocument.RootElement.GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(0, compactDocument.RootElement.GetProperty("limit_omitted_count").GetInt32()); + Assert.Equal(5, compactDocument.RootElement.GetProperty("total_count").GetInt32()); + Assert.False(compactDocument.RootElement.GetProperty("truncated").GetBoolean()); + + var (emptyCompactExitCode, emptyCompactStdout, emptyCompactStderr) = CaptureConsole(() => + ProgramRunner.Run( + [ + "search", + "Issue4843DefinitelyMissing", + "--db", dbPath, + "--exact-substring", + "--format", "compact", + "--sample", "5", + "--limit", "20", + ], + _jsonOptions, + "test")); + + Assert.Equal(CommandExitCodes.Success, emptyCompactExitCode); + Assert.Equal(string.Empty, emptyCompactStderr); + using var emptyCompactDocument = ParseJsonOutput(emptyCompactStdout); + Assert.Equal(0, emptyCompactDocument.RootElement.GetProperty("source_total").GetInt32()); + Assert.Equal(0, emptyCompactDocument.RootElement.GetProperty("selected_total").GetInt32()); + Assert.Equal(0, emptyCompactDocument.RootElement.GetProperty("returned").GetInt32()); + Assert.Single(emptyCompactDocument.RootElement.GetProperty("selectors").EnumerateArray()); + + var (envelopeExitCode, envelopeStdout, envelopeStderr) = CaptureConsole(() => + ProgramRunner.Run( + [ + "search", + "Console.WriteLine", + "--db", dbPath, + "--exact-substring", + "--json=array", + "--json-envelope", + "--sample", "5", + "--limit", "20", + ], + _jsonOptions, + "test")); + + Assert.Equal(CommandExitCodes.Success, envelopeExitCode); + Assert.Equal(string.Empty, envelopeStderr); + using var envelopeDocument = ParseJsonOutput(envelopeStdout); + var envelopeMetadata = envelopeDocument.RootElement.GetProperty("metadata"); + Assert.Equal(5, envelopeMetadata.GetProperty("total_count").GetInt32()); + Assert.Equal(5, envelopeMetadata.GetProperty("returned_count").GetInt32()); + var envelopeTerminal = envelopeMetadata.GetProperty("stream_terminal"); + Assert.Equal(126, envelopeTerminal.GetProperty("source_total").GetInt32()); + Assert.True(envelopeTerminal.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(5, envelopeTerminal.GetProperty("selected_total").GetInt32()); + Assert.Equal(5, envelopeTerminal.GetProperty("returned").GetInt32()); + Assert.Equal(121, envelopeTerminal.GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(0, envelopeTerminal.GetProperty("limit_omitted_count").GetInt32()); + var (combinedExitCode, combinedStdout, combinedStderr) = CaptureConsole(() => QueryCommandRunner.RunSearch( [ @@ -7498,6 +7581,51 @@ public void RunSearch_RecipeSelectionAccountingReportsSourceSelectorsAndLimit_Is Assert.Equal(5, contextSelectors[1].GetProperty("sample_size").GetInt32()); Assert.Equal("deterministic_seeded_v1", contextSelectors[1].GetProperty("sample_mode").GetString()); Assert.Equal(0, contextSelectors[1].GetProperty("seed").GetInt32()); + + var (guardedExitCode, guardedStdout, guardedStderr) = CaptureConsole(() => + QueryCommandRunner.RunSearch( + [ + "Console.WriteLine", + "--db", dbPath, + "--exact-substring", + "--json", + "--sample", "5", + "--limit", "20", + "--reject-after", "Issue4843DefinitelyMissingGuard", + ], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, guardedExitCode); + Assert.Equal(string.Empty, guardedStderr); + var guardedLines = guardedStdout.Split( + '\n', + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + using var guardedTerminal = JsonDocument.Parse(guardedLines[^1]); + Assert.Equal(20, guardedTerminal.RootElement.GetProperty("source_total").GetInt32()); + Assert.False(guardedTerminal.RootElement.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal(20, guardedTerminal.RootElement.GetProperty("source_total_lower_bound").GetInt32()); + + var (guardedRecipeExitCode, guardedRecipeStdout, guardedRecipeStderr) = CaptureConsole(() => + QueryCommandRunner.RunSearch( + [ + "--recipe", "risky-code/raw-diagnostic-echo", + "--db", dbPath, + "--json", + "--sample", "5", + "--limit", "20", + "--reject-after", "Issue4843DefinitelyMissingGuard", + ], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, guardedRecipeExitCode); + Assert.Equal(string.Empty, guardedRecipeStderr); + using var guardedRecipeDocument = ParseJsonOutput(guardedRecipeStdout); + var guardedRecipeQuery = Assert.Single( + guardedRecipeDocument.RootElement.GetProperty("queries").EnumerateArray()); + Assert.False(guardedRecipeQuery.GetProperty("source_total_authoritative").GetBoolean()); + Assert.Equal( + guardedRecipeQuery.GetProperty("source_total").GetInt32(), + guardedRecipeQuery.GetProperty("source_total_lower_bound").GetInt32()); } finally { @@ -7575,6 +7703,7 @@ public void RunSearch_RecipeRejectsRowSelectionForNonRowOutputs_Issue4759() (Args: new[] { "--recipe", "risky-code/raw-diagnostic-echo", "--format", "count", "--sample", "1" }, Expected: "recipe row-selection controls"), (Args: new[] { "--recipe", "risky-code/raw-diagnostic-echo", "--count-by", "path", "--first-per-file" }, Expected: "recipe row-selection controls"), (Args: new[] { "--recipe", "risky-code/raw-diagnostic-echo", "--format", "compact", "--summary-only", "--sample", "1" }, Expected: "recipe row-selection controls"), + (Args: new[] { "--recipe", "risky-code/raw-diagnostic-echo", "--json", "--results-only", "--sample", "1" }, Expected: "recipe row-selection controls"), (Args: new[] { "--recipe", "risky-code/raw-diagnostic-echo", "--per-file-limit", "1" }, Expected: "--per-file-limit is not supported with --recipe"), (Args: new[] { "--recipe", "risky-code/raw-diagnostic-echo", "--cursor", "0:1:1", "--sample", "1" }, Expected: "cannot be combined with --cursor"), (Args: new[] { "--recipe", "risky-code/raw-diagnostic-echo", "--cursor", "0:1:1", "--first-per-file" }, Expected: "cannot be combined with --cursor"), @@ -7618,6 +7747,31 @@ public void RunAudit_CountRejectsRowSelectorsBeforeExecution_Issue4843() } } + [Fact] + public void RunSearch_RejectsRowSelectorsWhenOutputCannotReportAccounting_Issue4843() + { + var cases = new[] + { + (Args: new[] { "needle", "--count", "--sample", "1" }, Expected: "cannot be combined with count or aggregation"), + (Args: new[] { "needle", "--count-by", "file", "--first-per-file" }, Expected: "cannot be combined with count or aggregation"), + (Args: new[] { "--named-query", "one=needle", "--sample", "1" }, Expected: "not supported with --named-query"), + (Args: new[] { "--list-recipes", "--first-per-file" }, Expected: "not supported with --list-recipes"), + (Args: new[] { "needle", "--json", "--results-only", "--sample", "1" }, Expected: "cannot be combined with --results-only"), + (Args: new[] { "needle", "--json=array", "--sample", "1" }, Expected: "metadata-free --json=array"), + (Args: new[] { "needle", "--format", "csv", "--sample", "1" }, Expected: "only supported by text, JSON, compact, and issue-drafts"), + }; + + foreach (var testCase in cases) + { + var (exitCode, stdout, stderr) = CaptureConsole(() => + QueryCommandRunner.RunSearch(testCase.Args, _jsonOptions)); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal(string.Empty, stdout); + Assert.Contains(testCase.Expected, stderr, StringComparison.Ordinal); + } + } + [Fact] public void RunSearch_RecipeCursorRequiresSingleChildQuery_Issue3392() { From 37fe375e043f06e26fc57237dee6f8f4f5ed7169 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 28 Jul 2026 02:24:20 +0900 Subject: [PATCH 3/3] Fix selector accounting edge cases (#4843) --- DEVELOPER_GUIDE.md | 2 + USER_GUIDE.md | 11 ++ changelog.d/unreleased/4843.fixed.md | 5 +- .../Cli/JsonEnvelopeWrapper.Bounded.cs | 51 +++++- .../Cli/QueryCommandRunner.SearchRecipes.cs | 28 ++- src/CodeIndex/Cli/SearchAuditRecipes.cs | 24 ++- .../QueryCommandRunnerSearchTests.cs | 165 +++++++++++++++++- 7 files changed, 276 insertions(+), 10 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 458c81c3ca..a3d94d06c1 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1558,6 +1558,7 @@ access. | Bounded response edge cases | `impact` applies the cursor offset only to the selected nested collection so definition pages do not repeat or alter caller/fallback mode. Plain `map --compact` preserves its established section arrays and truncation payload; a collection projection is rejected when `--summary-only` or an excluding `--sections` filter would remove it. Explicit definition body fields override compact defaults. Profile and verbose records are moved into `metadata.stream_control_records`, and parser/capture failures emit an error envelope only when it fits the active hard byte cap. | | `--count --json` envelope | Count-only JSON for `search`, `definition`, `references`, `callers`, `callees`, `symbols`, `files`, `find`, `impact`, and `unused` is a single automation-oriented object. It always includes `count`, applied `query_context`, freshness metadata (`indexed_file_count`, `indexed_at`, `freshness_available`), and trust flags `degraded` / `authoritative_count`; commands with matched-file totals also include `files` and the older `file_count` compatibility alias. `file_count` carries the same value as `files`, remains for compatibility, and is not scheduled for removal before the next major release. `unused --count --json` also includes `returned_bucket_counts`, `returned_contract_domain_counts`, and `summary.by_bucket` / `summary.by_confidence` / `summary.by_contract_domain`. `authoritative_count=false` means a readiness or graph/exact trust signal made the count non-authoritative, while the freshness fields describe the indexed snapshot used for the count. | | Search row selection | Row-producing plain-search and recipe paths share `ApplySearchOutputSelection`: `--first-per-file` and fixed-seed deterministic `--sample` run before the effective per-query / remaining total limit. Sample fetch envelopes are sized from at least the requested sample target. Aggregate/compact query DTOs, plain compact roots, run summaries, issue-draft source DTOs, NDJSON terminals, and bounded array-envelope stream terminals expose `source_total`, `selected_total`, `returned`, `selector_omitted_count`, and `limit_omitted_count`; `source_total_authoritative` / `source_total_lower_bound` distinguish complete populations from bounded observations. Guard filters, origin/facet post-filters, exhausted candidate windows, and recipe file-reject post-filters force lower-bound authority. Their ordered `selectors` entries preserve each stage's input/output/omission counts plus sample size, mode, and seed, while nullable `selection_reason` / `selection_omitted_count` remain compatibility summaries. Bounded plain-search selection is computed once and its selected page is reused by compact/envelope serialization. Search `query_context.row_selectors` records the applied selector configuration. Selection-only omission updates matched/omitted lower bounds without setting `truncated`, `has_more`, or `next_cursor`. When a later limit truncates selected rows, limit truncation remains visible but `next_cursor` is suppressed because raw database cursors cannot preserve selector state; incoming `--cursor` values are rejected with either selector for the same reason. Generated compact and issue-draft replay commands retain the selector. Count, aggregation, named-query, recipe-list, results-only, metadata-free array, unsupported formatted, and summary-only compact shapes reject `--first-per-file` / `--sample`, while every recipe shape rejects grouped-only `--per-file-limit`. | +| Search selection edge cases | Issue-draft roots independently retain per-query `selection_accounting`, including zero-draft and exhausted-total-limit queries. Byte-bounded compact and array envelopes rewrite `returned` to the emitted row count while preserving logical `limit_omitted_count`; hard-cap omissions remain separate in `metadata.byte_limit_omitted_count`. | | Ad-hoc issue-draft selection | `search --format issue-drafts` reads the complete filtered ad-hoc population, then applies `--first-per-file`, deterministic `--sample`, and `min(--limit, --total-limit)` in that order. Guarded searches retain their finite candidate inspection contract: `source_total_count` is omitted, `source_minimum_count` reports the observed lower bound, `source_total_count_authoritative=false`, `source_fetch_limit` reports the bounded fetch, and `truncated=true` preserves incomplete-population state. Existing `result_count`, `result_limit`, `omitted_count`, and `truncated` fields describe the returned selection accurately; additive `source_total_count`, `returned_count`, `limit_per_query`, `total_limit`, `first_per_file`, and `sample` fields make the applied contract auditable. Replay commands are serialized from normalized parsed options, use POSIX-safe single-quote escaping, and retain raw/exact/prefix modes, path/language/facet/guard filters, selection controls, evidence formatting, duplicate preflight, and issue hints. | | Recipe SARIF | `search --recipe --format sarif` emits one result per bounded recipe result. Rule IDs use `recipe/query`; standard `fingerprints.cdidx/v1` values are derived from the normalized source location; result properties preserve recipe/query identity, severity, confidence, and per-query truncation; run properties preserve scope, applied result limits, aggregate counts, and conservative omitted-result metadata. Bound SARIF with `--limit` / `--total-limit`; row selectors such as `--sample`, `--first-per-file`, and `--per-file-limit` are rejected instead of being silently ignored. Recipe severity maps `critical` / `high` to `error`, `medium` to `warning`, and `low` / `info` to `note`. | | Recipe classifier output | Recipe run JSON may add `audit_classifications` to individual `CompactSearchResult` rows when a recipe classifier can classify the hit, and query/count payloads may add `classifier_counts` when classified rows are present. These fields are additive; use them to separate triage domains such as DTO/result-wrapper `.Result` properties versus Task/ValueTask blocking waits without changing the raw search query. | @@ -4749,6 +4750,7 @@ help はすべてこのレジストリを参照します。field 名は大文字 | bounded 応答の edge case | `impact` は選択された nested collection だけに cursor offset を適用するため、definition page の重複や caller / fallback mode の変化を防ぎます。通常の `map --compact` は既存の section array と truncation payload を維持し、collection projection が `--summary-only` または除外する `--sections` filter で失われる組み合わせは拒否します。明示的な definition body field は compact default より優先します。profile / verbose record は `metadata.stream_control_records` へ移し、parser / capture failure の error envelope は active な hard byte cap に収まる場合だけ出力します。 | | `--count --json` envelope | `search`、`definition`、`references`、`callers`、`callees`、`symbols`、`files`、`find`、`impact`、`unused` の count-only JSON は単一の自動化向け object です。常に `count`、適用済み `query_context`、freshness metadata（`indexed_file_count`、`indexed_at`、`freshness_available`）、trust flag の `degraded` / `authoritative_count` を含みます。matched-file total を持つ command は `files` と古い互換 alias の `file_count` も含みます。`file_count` は `files` と同じ値を持つ互換用 field として残り、少なくとも次の major release までは削除予定はありません。`unused --count --json` は `returned_bucket_counts`、`returned_contract_domain_counts`、`summary.by_bucket` / `summary.by_confidence` / `summary.by_contract_domain` も含みます。`authoritative_count=false` は readiness または graph/exact trust signal により count が authoritative ではないことを示し、freshness field は count に使った index snapshot を説明します。 | | search row selection | row を返す plain search / recipe path は `ApplySearchOutputSelection` を共有し、`--first-per-file` と固定 seed の決定的な `--sample` を、有効な query ごとの limit / 残り total limit より先に適用します。sample 用 fetch envelope は少なくとも要求 sample 数を基準に sizing します。aggregate / compact の query DTO、plain compact root、run summary、issue-draft の source DTO、NDJSON terminal、bounded array envelope の stream terminal は `source_total`、`selected_total`、`returned`、`selector_omitted_count`、`limit_omitted_count` を公開し、`source_total_authoritative` / `source_total_lower_bound` で完全な population と bounded な観測を区別します。guard filter、origin / facet の後段 filter、candidate window の枯渇、recipe の file-reject 後段 filter は lower-bound authority にします。適用順の `selectors` entry は各段階の input / output / omission count と sample の size / mode / seed を保持し、nullable な `selection_reason` / `selection_omitted_count` は互換用 summary として維持します。bounded plain-search selection は一度だけ計算し、その selected page を compact / envelope serialize で再利用します。search の `query_context.row_selectors` は適用済み selector 設定を記録します。selection だけによる省略は matched / omitted の lower bound を更新しますが、`truncated`、`has_more`、`next_cursor` は設定しません。後続の limit が選択済み row を truncate する場合、limit truncation は表示しますが raw database cursor は selector state を保持できないため `next_cursor` を抑止し、同じ理由で selector と受け取った `--cursor` の併用も拒否します。compact / issue-draft の生成 replay command は selector を保持します。count、aggregation、named-query、recipe-list、results-only、metadata を持たない array、非対応 formatted、summary-only compact の shape は `--first-per-file` / `--sample` を拒否し、すべての recipe shape は grouped 専用の `--per-file-limit` を拒否します。 | +| search selection の edge case | issue-draft の root は query ごとの `selection_accounting` を独立して保持するため、draft が 0 件の場合や total limit を使い切った query も accounting を失いません。byte 上限付き compact / array envelope は `returned` を実際の出力 row 数へ更新し、論理的な `limit_omitted_count` を保持します。hard cap による省略は `metadata.byte_limit_omitted_count` で別に報告します。 | | ad-hoc issue-draft selection | `search --format issue-drafts` は filter 済みの ad-hoc 母集団全体を読み、`--first-per-file`、決定的な `--sample`、`min(--limit, --total-limit)` の順に適用します。guard 付き検索は有限の candidate inspection 契約を維持し、`source_total_count` を省略し、観測下限を `source_minimum_count`、非 authoritative 状態を `source_total_count_authoritative=false`、bounded fetch を `source_fetch_limit` で報告し、母集団が未完了であることを `truncated=true` で保持します。既存の `result_count`、`result_limit`、`omitted_count`、`truncated` field は返却 selection を正確に表し、additive な `source_total_count`、`returned_count`、`limit_per_query`、`total_limit`、`first_per_file`、`sample` field により適用済み契約を監査できます。replay command は正規化済み parse option から serialize し、POSIX-safe な単一引用符 escape を使い、raw / exact / prefix mode、path / language / facet / guard filter、selection control、evidence formatting、duplicate preflight、issue hint を維持します。 | | Recipe SARIF | `search --recipe --format sarif` は、上限付き recipe result ごとに result を1件出力します。rule ID は `recipe/query` を使い、標準の `fingerprints.cdidx/v1` は正規化済み source location から導出します。result properties は recipe/query identity、severity、confidence、query ごとの truncation を保持し、run properties は scope、適用済み result limit、集計 count、保守的な omitted-result metadata を保持します。SARIF の上限には `--limit` / `--total-limit` を使い、`--sample`、`--first-per-file`、`--per-file-limit` のような row selector は黙って無視せず拒否します。recipe severity は `critical` / `high` を `error`、`medium` を `warning`、`low` / `info` を `note` に対応付けます。 | | Recipe classifier output | recipe classifier が hit を分類できる場合、recipe run JSON は個別の `CompactSearchResult` row に `audit_classifications` を追加することがあり、分類済み row がある query / count payload は `classifier_counts` を追加することがあります。これらは additive field です。raw search query を変えずに、DTO / result-wrapper の `.Result` property と Task / ValueTask の blocking wait などの triage domain を分離するために使います。 | diff --git a/USER_GUIDE.md b/USER_GUIDE.md index aa1e2e54fa..6aeadaa70f 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1393,6 +1393,12 @@ and compact query objects, plain compact roots, run summaries, issue-draft `limit_omitted_count`. Their `selectors` array records each applied selector in execution order, including per-stage input/output/omission counts and the sample size, mode, and seed. +Issue-draft roots also expose per-query `selection_accounting`, so selector +accounting remains available when no draft is emitted or a cross-query total +limit leaves a selected query with zero returned rows. For byte-bounded compact +and array envelopes, `returned` reflects the rows that fit the final envelope; +logical `limit_omitted_count` remains unchanged, while +`metadata.byte_limit_omitted_count` reports rows removed by the hard byte cap. `source_total_authoritative` says whether the bounded fetch observed the whole source population; guard filters, origin/facet post-filters, bounded candidate windows, and recipe file-reject post-filters conservatively produce @@ -4535,6 +4541,11 @@ array envelope の `metadata.stream_terminal` と同様に、`source_total`、`s `returned`、`selector_omitted_count`、`limit_omitted_count` を分けて返します。 `selectors` array は適用順の各 selector について、 各段階の入力件数、出力件数、省略件数、および sample の size / mode / seed を記録します。 +issue-draft の root も query ごとの `selection_accounting` を公開するため、draft が 0 件の場合や +query 全体の total limit により選択済み query の返却 row が 0 件になった場合も selector accounting +を保持します。byte 上限付き compact / array envelope の `returned` は最終 envelope に収まった +row 数を表します。論理的な `limit_omitted_count` は変更せず、hard byte cap で除外した row 数は +`metadata.byte_limit_omitted_count` で別に報告します。 bounded fetch が source population 全体を観測できたかは `source_total_authoritative` で示し、 guard filter、origin / facet の後段 filter、bounded candidate window、recipe の file-reject 後段 filter がある場合は保守的に `source_total_authoritative=false` と diff --git a/changelog.d/unreleased/4843.fixed.md b/changelog.d/unreleased/4843.fixed.md index c32bb262ea..1893f8a7d7 100644 --- a/changelog.d/unreleased/4843.fixed.md +++ b/changelog.d/unreleased/4843.fixed.md @@ -3,6 +3,7 @@ category: fixed issues: - 4843 affected: + - src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs - src/CodeIndex/Cli/JsonOutputContracts.cs - src/CodeIndex/Cli/QueryCommandRunner.Ndjson.cs - src/CodeIndex/Cli/QueryCommandRunner.ResultEnvelopes.cs @@ -18,8 +19,8 @@ affected: ## English -- **Sampling and first-per-file output now reports the source, selected, and returned populations separately (#4843)** — recipe JSON, plain compact output, array envelopes, issue drafts, and NDJSON terminals now expose per-selector accounting, limit omissions, deterministic sample mode/seed, and bounded-source authority. Search query context lists every applied selector; guard and post-filtered populations are reported as lower bounds; count, aggregation, named-query, recipe-list, results-only, and metadata-free array modes reject selectors with a stable usage error instead of silently ignoring them. +- **Sampling and first-per-file output now reports the source, selected, and returned populations separately (#4843)** — recipe JSON, plain compact output, array envelopes, issue drafts, and NDJSON terminals now expose per-selector accounting, limit omissions, deterministic sample mode/seed, and bounded-source authority. Issue-draft roots retain per-query accounting even when no draft is returned, and byte-bounded envelopes report their actually emitted selector rows separately from byte-cap omissions. Search query context lists every applied selector; guard and post-filtered populations are reported as lower bounds; count, aggregation, named-query, recipe-list, results-only, and metadata-free array modes reject selectors with a stable usage error instead of silently ignoring them. ## 日本語 -- **sample / first-per-file 出力で選択前・選択後・返却後の件数を分けて報告するようになりました (#4843)** — recipe JSON、plain compact 出力、array envelope、issue draft、NDJSON terminal は selector ごとの件数、limit による省略、決定的 sample の mode / seed、bounded な source 件数の authority を公開します。search の query context は適用済み selector をすべて列挙し、guard / 後段 filter 付きの population は lower bound として報告します。count、aggregation、named-query、recipe-list、results-only、metadata を持たない array mode は selector を黙って無視せず、安定した usage error で拒否します。 +- **sample / first-per-file 出力で選択前・選択後・返却後の件数を分けて報告するようになりました (#4843)** — recipe JSON、plain compact 出力、array envelope、issue draft、NDJSON terminal は selector ごとの件数、limit による省略、決定的 sample の mode / seed、bounded な source 件数の authority を公開します。issue-draft の root は draft が返らない場合も query ごとの accounting を保持し、byte 上限付き envelope は実際に返した selector row と byte cap による省略を分けて報告します。search の query context は適用済み selector をすべて列挙し、guard / 後段 filter 付きの population は lower bound として報告します。count、aggregation、named-query、recipe-list、results-only、metadata を持たない array mode は selector を黙って無視せず、安定した usage error で拒否します。 diff --git a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs index 4dc055135e..0a0d051321 100644 --- a/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs +++ b/src/CodeIndex/Cli/JsonEnvelopeWrapper.Bounded.cs @@ -336,6 +336,10 @@ JsonObject BuildCandidate(int count) var results = new JsonArray(); for (var i = 0; i < count; i++) results.Add(pageItems[i]?.DeepClone()); + var adjustedStreamTerminal = AdjustSearchSelectionAccountingForBoundedRows( + command, + streamTerminal, + count); var envelope = BuildEnvelope( command, queryNormalized, @@ -346,7 +350,7 @@ JsonObject BuildCandidate(int count) results, exitCode, error: commandError is null ? null : (JsonObject)commandError.DeepClone(), - streamTerminal: streamTerminal, + streamTerminal: adjustedStreamTerminal, streamControlRecords: streamControlRecords); var metadata = (JsonObject)envelope["metadata"]!; metadata["result_stable_at"] = snapshot.ResultStableAt; @@ -415,6 +419,7 @@ JsonObject BuildCandidate(int count) && extraction.SourcePayload is not null) { return BuildBackwardCompatibleCompactEnvelope( + command, extraction.SourcePayload, envelope, results, @@ -441,7 +446,7 @@ JsonObject BuildCandidate(int count) var requestedCount = pageItems.Count; var candidate = BuildCandidate(requestedCount); - var candidateJson = candidate.ToJsonString(jsonOptions); + var candidateJson = SerializeBoundedEnvelope(candidate, jsonOptions); if (!controls.MaxJsonBytes.HasValue || JsonFitsResponseBudget(candidateJson, controls.MaxJsonBytes.Value)) { emittedJson = candidateJson; @@ -457,7 +462,7 @@ JsonObject BuildCandidate(int count) { var mid = low + ((high - low) / 2); var current = BuildCandidate(mid); - var currentJson = current.ToJsonString(jsonOptions); + var currentJson = SerializeBoundedEnvelope(current, jsonOptions); if (JsonFitsResponseBudget(currentJson, controls.MaxJsonBytes.Value)) { best = current; @@ -477,6 +482,21 @@ JsonObject BuildCandidate(int count) return best; } + private static string SerializeBoundedEnvelope(JsonNode node, JsonSerializerOptions jsonOptions) + { + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream, new JsonWriterOptions + { + Encoder = jsonOptions.Encoder, + Indented = jsonOptions.WriteIndented, + })) + { + node.WriteTo(writer); + } + + return Encoding.UTF8.GetString(stream.ToArray()); + } + private static bool JsonFitsResponseBudget(string json, int maxJsonBytes) => Encoding.UTF8.GetByteCount(json) + Encoding.UTF8.GetByteCount(Environment.NewLine) <= maxJsonBytes; @@ -537,6 +557,7 @@ private static void PromoteEmptyLegacyCompactPayload( } private static JsonObject BuildBackwardCompatibleCompactEnvelope( + string command, JsonObject sourcePayload, JsonObject envelope, JsonArray results, @@ -551,7 +572,10 @@ private static JsonObject BuildBackwardCompatibleCompactEnvelope( ResponseSnapshot snapshot, string? primaryCollection) { - var compatible = (JsonObject)sourcePayload.DeepClone(); + var compatible = AdjustSearchSelectionAccountingForBoundedRows( + command, + sourcePayload, + returnedCount) ?? (JsonObject)sourcePayload.DeepClone(); var collectionName = primaryCollection ?? "results"; compatible[collectionName] = results.DeepClone(); compatible["metadata"] = envelope["metadata"]!.DeepClone(); @@ -587,6 +611,25 @@ private static JsonObject BuildBackwardCompatibleCompactEnvelope( return compatible; } + private static JsonObject? AdjustSearchSelectionAccountingForBoundedRows( + string command, + JsonObject? payload, + int returnedCount) + { + if (payload is null + || !string.Equals(command, "search", StringComparison.Ordinal) + || payload["selectors"] is not JsonArray + || !payload.ContainsKey("returned")) + { + return payload; + } + + var adjusted = (JsonObject)payload.DeepClone(); + adjusted["returned"] = JsonNode.Parse( + Math.Max(0, returnedCount).ToString(CultureInfo.InvariantCulture)); + return adjusted; + } + private static JsonObject BuildBackwardCompatibleMapCompactEnvelope( JsonObject sourcePayload, JsonObject envelope) diff --git a/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs b/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs index 9bc65c1b41..324a1117da 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.SearchRecipes.cs @@ -1206,7 +1206,8 @@ private static int RunSearchRecipeIssueDrafts( preflight.OpenIssueCount, options.DuplicateConfidence, options.DuplicateThreshold), - drafts), + drafts, + BuildSearchIssueDraftSelectionAccounting(recipe.Name, queryResults)), CliJsonSerializerContextFactory.Create(jsonOptions).SearchIssueDraftExportJsonResult); return WriteJsonObjectWithOptionalByteLimit( json, @@ -1409,7 +1410,8 @@ private static int RunSearchIssueDrafts( preflight.OpenIssueCount, options.DuplicateConfidence, options.DuplicateThreshold), - drafts), + drafts, + BuildSearchIssueDraftSelectionAccounting(null, [queryResult])), CliJsonSerializerContextFactory.Create(jsonOptions).SearchIssueDraftExportJsonResult); return WriteJsonObjectWithOptionalByteLimit( json, @@ -1419,6 +1421,28 @@ private static int RunSearchIssueDrafts( }); } + private static List? BuildSearchIssueDraftSelectionAccounting( + string? recipeName, + IReadOnlyList queryResults) + { + var accounting = queryResults + .Where(query => query.Selectors.Count > 0) + .Select(query => new SearchIssueDraftSelectionAccountingJsonResult( + recipeName, + recipeName is null ? null : query.Name, + query.Query, + query.SourceTotal, + query.SourceTotalAuthoritative, + query.SourceTotalLowerBound, + query.SelectedTotal, + query.Returned, + query.SelectorOmittedCount, + query.LimitOmittedCount, + query.Selectors)) + .ToList(); + return accounting.Count == 0 ? null : accounting; + } + private static int GetAdHocIssueDraftResultLimit(QueryCommandOptions options) => options.TotalLimit.HasValue ? Math.Min(options.Limit, options.TotalLimit.Value) diff --git a/src/CodeIndex/Cli/SearchAuditRecipes.cs b/src/CodeIndex/Cli/SearchAuditRecipes.cs index dd22147bb9..62885edefe 100644 --- a/src/CodeIndex/Cli/SearchAuditRecipes.cs +++ b/src/CodeIndex/Cli/SearchAuditRecipes.cs @@ -4605,7 +4605,29 @@ internal sealed record SearchIssueDraftExportJsonResult( SearchRecipeQueryFreshnessJsonResult? QueryFreshness, [property: JsonPropertyName("count")] int Count, [property: JsonPropertyName("duplicate_preflight")] SuggestionIssueDraftPreflightSummaryJsonResult DuplicatePreflight, - [property: JsonPropertyName("drafts")] List Drafts); + [property: JsonPropertyName("drafts")] List Drafts, + [property: JsonPropertyName("selection_accounting")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + List? SelectionAccounting); + +internal sealed record SearchIssueDraftSelectionAccountingJsonResult( + [property: JsonPropertyName("recipe")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + string? Recipe, + [property: JsonPropertyName("query_name")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + string? QueryName, + [property: JsonPropertyName("query")] string Query, + [property: JsonPropertyName("source_total")] int SourceTotal, + [property: JsonPropertyName("source_total_authoritative")] bool SourceTotalAuthoritative, + [property: JsonPropertyName("source_total_lower_bound")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + int? SourceTotalLowerBound, + [property: JsonPropertyName("selected_total")] int SelectedTotal, + [property: JsonPropertyName("returned")] int Returned, + [property: JsonPropertyName("selector_omitted_count")] int SelectorOmittedCount, + [property: JsonPropertyName("limit_omitted_count")] int LimitOmittedCount, + [property: JsonPropertyName("selectors")] List Selectors); internal sealed record SearchIssueDraftJsonResult( [property: JsonPropertyName("draft_id")] string DraftId, diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs index a251bc97d1..d653fa0f10 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerSearchTests.cs @@ -7367,7 +7367,7 @@ public void RunSearch_RecipeSelectionAccountingReportsSourceSelectorsAndLimit_Is var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); for (var fileIndex = 0; fileIndex < 6; fileIndex++) { - var path = $"src/selection-{fileIndex}.cs"; + var path = $"src/selection-{fileIndex}-{new string('x', 80)}.cs"; TestProjectHelper.InsertIndexedFile(dbPath, path, "csharp", "Console.WriteLine(ex.Message);\n"); ReplaceIndexedChunks( dbPath, @@ -7495,6 +7495,66 @@ public void RunSearch_RecipeSelectionAccountingReportsSourceSelectorsAndLimit_Is Assert.Equal(121, envelopeTerminal.GetProperty("selector_omitted_count").GetInt32()); Assert.Equal(0, envelopeTerminal.GetProperty("limit_omitted_count").GetInt32()); + var (byteCompactExitCode, byteCompactStdout, byteCompactStderr) = CaptureConsole(() => + ProgramRunner.Run( + [ + "search", + "Console.WriteLine", + "--db", dbPath, + "--exact-substring", + "--format", "compact", + "--sample", "20", + "--limit", "20", + "--max-json-bytes", "3000", + ], + _jsonOptions, + "test")); + + Assert.Equal(CommandExitCodes.Success, byteCompactExitCode); + Assert.Equal(string.Empty, byteCompactStderr); + using var byteCompactDocument = ParseJsonOutput(byteCompactStdout); + var byteCompactRoot = byteCompactDocument.RootElement; + var byteCompactReturned = byteCompactRoot.GetProperty("results").GetArrayLength(); + Assert.InRange(byteCompactReturned, 1, 19); + Assert.Equal(20, byteCompactRoot.GetProperty("selected_total").GetInt32()); + Assert.Equal(byteCompactReturned, byteCompactRoot.GetProperty("returned").GetInt32()); + Assert.Equal(0, byteCompactRoot.GetProperty("limit_omitted_count").GetInt32()); + Assert.Equal( + 20 - byteCompactReturned, + byteCompactRoot.GetProperty("metadata").GetProperty("byte_limit_omitted_count").GetInt32()); + + var (byteEnvelopeExitCode, byteEnvelopeStdout, byteEnvelopeStderr) = CaptureConsole(() => + ProgramRunner.Run( + [ + "search", + "Console.WriteLine", + "--db", dbPath, + "--exact-substring", + "--json=array", + "--json-envelope", + "--fields", "path", + "--sample", "20", + "--limit", "20", + "--max-json-bytes", "2500", + ], + _jsonOptions, + "test")); + + Assert.Equal(CommandExitCodes.Success, byteEnvelopeExitCode); + Assert.Equal(string.Empty, byteEnvelopeStderr); + using var byteEnvelopeDocument = ParseJsonOutput(byteEnvelopeStdout); + var byteEnvelopeRoot = byteEnvelopeDocument.RootElement; + var byteEnvelopeReturned = byteEnvelopeRoot.GetProperty("results").GetArrayLength(); + Assert.InRange(byteEnvelopeReturned, 1, 19); + var byteEnvelopeMetadata = byteEnvelopeRoot.GetProperty("metadata"); + var byteEnvelopeTerminal = byteEnvelopeMetadata.GetProperty("stream_terminal"); + Assert.Equal(20, byteEnvelopeTerminal.GetProperty("selected_total").GetInt32()); + Assert.Equal(byteEnvelopeReturned, byteEnvelopeTerminal.GetProperty("returned").GetInt32()); + Assert.Equal(0, byteEnvelopeTerminal.GetProperty("limit_omitted_count").GetInt32()); + Assert.Equal( + 20 - byteEnvelopeReturned, + byteEnvelopeMetadata.GetProperty("byte_limit_omitted_count").GetInt32()); + var (combinedExitCode, combinedStdout, combinedStderr) = CaptureConsole(() => QueryCommandRunner.RunSearch( [ @@ -7551,6 +7611,56 @@ public void RunSearch_RecipeSelectionAccountingReportsSourceSelectorsAndLimit_Is Assert.True(emptySelector.GetProperty("applied").GetBoolean()); Assert.Equal(0, emptySelector.GetProperty("omitted_count").GetInt32()); + var (emptyDraftExitCode, emptyDraftStdout, emptyDraftStderr) = CaptureConsole(() => + QueryCommandRunner.RunSearch( + [ + "Issue4843DefinitelyMissing", + "--db", dbPath, + "--exact-substring", + "--format", "issue-drafts", + "--sample", "5", + "--path", "missing/**", + ], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, emptyDraftExitCode); + Assert.Equal(string.Empty, emptyDraftStderr); + using var emptyDraftDocument = ParseJsonOutput(emptyDraftStdout); + Assert.Empty(emptyDraftDocument.RootElement.GetProperty("drafts").EnumerateArray()); + var emptyDraftAccounting = Assert.Single( + emptyDraftDocument.RootElement.GetProperty("selection_accounting").EnumerateArray()); + Assert.Equal(0, emptyDraftAccounting.GetProperty("source_total").GetInt32()); + Assert.Equal(0, emptyDraftAccounting.GetProperty("selected_total").GetInt32()); + Assert.Equal(0, emptyDraftAccounting.GetProperty("returned").GetInt32()); + Assert.Equal(0, emptyDraftAccounting.GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(0, emptyDraftAccounting.GetProperty("limit_omitted_count").GetInt32()); + Assert.Single(emptyDraftAccounting.GetProperty("selectors").EnumerateArray()); + + var (emptyRecipeDraftExitCode, emptyRecipeDraftStdout, emptyRecipeDraftStderr) = CaptureConsole(() => + QueryCommandRunner.RunSearch( + [ + "--recipe", "risky-code/raw-diagnostic-echo", + "--db", dbPath, + "--format", "issue-drafts", + "--sample", "5", + "--path", "missing/**", + ], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, emptyRecipeDraftExitCode); + Assert.Equal(string.Empty, emptyRecipeDraftStderr); + using var emptyRecipeDraftDocument = ParseJsonOutput(emptyRecipeDraftStdout); + Assert.Empty(emptyRecipeDraftDocument.RootElement.GetProperty("drafts").EnumerateArray()); + var emptyRecipeDraftAccounting = Assert.Single( + emptyRecipeDraftDocument.RootElement.GetProperty("selection_accounting").EnumerateArray()); + Assert.Equal("risky-code", emptyRecipeDraftAccounting.GetProperty("recipe").GetString()); + Assert.Equal( + "raw-diagnostic-echo", + emptyRecipeDraftAccounting.GetProperty("query_name").GetString()); + Assert.Equal(0, emptyRecipeDraftAccounting.GetProperty("source_total").GetInt32()); + Assert.Equal(0, emptyRecipeDraftAccounting.GetProperty("selected_total").GetInt32()); + Assert.Equal(0, emptyRecipeDraftAccounting.GetProperty("returned").GetInt32()); + var (contextExitCode, contextStdout, contextStderr) = CaptureConsole(() => QueryCommandRunner.RunSearch( [ @@ -7633,6 +7743,59 @@ public void RunSearch_RecipeSelectionAccountingReportsSourceSelectorsAndLimit_Is } } + [Fact] + public void RunSearch_RecipeIssueDraftSelectionAccountingSurvivesExhaustedTotalLimit_Issue4843() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_search_issue_draft_selection_total_limit"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + TestProjectHelper.InsertIndexedFile( + dbPath, + "src/selection.cs", + "csharp", + "JsonDocument.Parse(payload);\nConsole.WriteLine(ex.Message);\n"); + + var (exitCode, stdout, stderr) = CaptureConsole(() => + QueryCommandRunner.RunSearch( + [ + "--recipe", "risky-code", + "--include-query", "unbounded-json-parse", + "--include-query", "raw-diagnostic-echo", + "--db", dbPath, + "--format", "issue-drafts", + "--first-per-file", + "--limit", "5", + "--total-limit", "1", + ], + _jsonOptions)); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + using var document = ParseJsonOutput(stdout); + Assert.Single(document.RootElement.GetProperty("drafts").EnumerateArray()); + var accounting = document.RootElement + .GetProperty("selection_accounting") + .EnumerateArray() + .ToArray(); + Assert.Equal(2, accounting.Length); + var exhausted = Assert.Single(accounting.Where(item => + item.GetProperty("query_name").GetString() == "raw-diagnostic-echo")); + Assert.Equal(1, exhausted.GetProperty("source_total").GetInt32()); + Assert.Equal(1, exhausted.GetProperty("selected_total").GetInt32()); + Assert.Equal(0, exhausted.GetProperty("returned").GetInt32()); + Assert.Equal(0, exhausted.GetProperty("selector_omitted_count").GetInt32()); + Assert.Equal(1, exhausted.GetProperty("limit_omitted_count").GetInt32()); + var selector = Assert.Single(exhausted.GetProperty("selectors").EnumerateArray()); + Assert.Equal("first_per_file", selector.GetProperty("mode").GetString()); + Assert.True(selector.GetProperty("applied").GetBoolean()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void RunSearch_RecipeSampleFetchesEnoughCandidatesBeforeLimit_Issue4759() {