diff --git a/.github/workflows/benchmark-regression.yml b/.github/workflows/benchmark-regression.yml
index d18bbee6..fefea3c0 100644
--- a/.github/workflows/benchmark-regression.yml
+++ b/.github/workflows/benchmark-regression.yml
@@ -20,6 +20,12 @@ on:
- 'CLAUDE.md'
- 'LICENSE'
workflow_dispatch:
+ inputs:
+ publish_baseline:
+ description: 'Publish this main-branch run as an intentional trusted baseline'
+ required: false
+ default: false
+ type: boolean
permissions:
contents: read
@@ -84,21 +90,44 @@ jobs:
echo "Baseline branch gh-benchmarks does not exist yet. Will be created on first push to main."
fi
- - name: Detect performance regression
+ - name: Resolve intended benchmark base
+ id: benchmark-base
+ shell: bash
+ run: |
+ base_sha="${{ github.event.pull_request.base.sha || github.event.before || github.sha }}"
+ if [[ ! "$base_sha" =~ ^[0-9a-f]{40}$ ]] || ! git cat-file -e "$base_sha^{commit}"; then
+ base_sha="$GITHUB_SHA"
+ fi
+ echo "sha=$base_sha" >> "$GITHUB_OUTPUT"
+
+ - name: Fetch trusted benchmark history
if: steps.check-baseline.outputs.exists == 'true'
- uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1
- with:
- name: 'FolderDiffIL4DotNet Performance'
- tool: 'benchmarkdotnet'
- output-file-path: BenchmarkDotNet.Artifacts/results/combined-report.json
- github-token: ${{ secrets.GITHUB_TOKEN }}
- auto-push: false
- alert-threshold: '200%'
- comment-on-alert: false
- fail-on-alert: true
- summary-always: true
- gh-pages-branch: gh-benchmarks
- benchmark-data-dir-path: dev/bench
+ run: |
+ git fetch --no-tags origin gh-benchmarks:refs/remotes/origin/gh-benchmarks
+ git show refs/remotes/origin/gh-benchmarks:dev/bench/data.js \
+ > BenchmarkDotNet.Artifacts/results/baseline-data.js
+
+ - name: Check evidence-based performance thresholds
+ if: steps.check-baseline.outputs.exists == 'true'
+ run: |
+ python3 scripts/check_benchmark_regressions.py \
+ --current-report BenchmarkDotNet.Artifacts/results/combined-report.json \
+ --history-data BenchmarkDotNet.Artifacts/results/baseline-data.js \
+ --policy benchmark-regression-policy.json \
+ --baseline-ancestor "${{ steps.benchmark-base.outputs.sha }}" \
+ --repository-root . \
+ --summary "$GITHUB_STEP_SUMMARY" \
+ --allow-failure "${{ github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.publish_baseline }}"
+
+ - name: Report missing benchmark baseline
+ if: steps.check-baseline.outputs.exists != 'true'
+ run: |
+ {
+ echo "## Performance regression gate"
+ echo
+ echo "⚠️ **WARMUP** — No trusted gh-benchmarks baseline exists yet."
+ echo "This run remains visible and will seed the baseline only from a trusted main-branch publication."
+ } >> "$GITHUB_STEP_SUMMARY"
- name: Upload benchmark results
if: always()
@@ -117,7 +146,11 @@ jobs:
publish-benchmark-baseline:
needs: benchmark-regression
- if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+ if: >-
+ (github.event_name == 'push' && github.ref == 'refs/heads/main') ||
+ (github.event_name == 'workflow_dispatch' &&
+ github.ref == 'refs/heads/main' &&
+ inputs.publish_baseline)
runs-on: ubuntu-latest
permissions:
contents: write
@@ -161,10 +194,6 @@ jobs:
output-file-path: BenchmarkDotNet.Artifacts/results/combined-report.json
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: true
- alert-threshold: '200%'
- comment-on-alert: false
- fail-on-alert: true
- summary-always: true
gh-pages-branch: gh-benchmarks
benchmark-data-dir-path: dev/bench
skip-fetch-gh-pages: ${{ steps.check-baseline.outputs.exists != 'true' }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d1fbdebe..736cec13 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -21,6 +21,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
#### Changed
+- **Evidence-based benchmark regression limits** — Replaced the single 200% emergency ceiling with a compatible-history median gate and per-stability-group warning/failure limits from 10/20% through 50/100%. The policy records seven same-definition `ubuntu-latest` samples, observed variance, threshold rationale, definition fingerprint inputs, sample minimum, and baseline revision. Actions summaries now show every pass, warning, failure, warmup, and excluded incompatible history entry; controlled fixtures protect calculation and exact boundaries. Trusted `main` publication remains automatic, with an explicit main-only manual mode for reviewed baseline resets. Affected: `.github/workflows/benchmark-regression.yml`, `benchmark-regression-policy.json`, `scripts/check_benchmark_regressions.py`, `scripts/tests/test_check_benchmark_regressions.py`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`, `doc/PERFORMANCE_GUIDE.md`, `doc/DEVELOPER_GUIDE.md`, `doc/TESTING_GUIDE.md`. Issue: #225.
+
- **Private vulnerability reporting and a streamlined security policy** — The repository now accepts confidential reports through GitHub private vulnerability reporting. `SECURITY.md` has one complete English section and one equivalent Japanese section covering supported versions, the direct private-reporting path, report contents, response targets, coordinated disclosure, and the preserved threat model. Regression coverage protects the reporting link, bilingual policy structure, and Japanese documentation anchors. Affected: `SECURITY.md`, `README.md`, `doc/DEVELOPER_GUIDE.md`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`, and the GitHub repository security setting. Issue: #223.
- **Release notes now use explicit dynamic comparison links** — The release workflow resolves the latest non-draft, non-prerelease stable GitHub Release by publication time, combines its tag with the current release tag to build the `Full Changelog` comparison URL, and appends the NuGet install/update commands in a consistent release-note template. Regression coverage keeps the release filtering, publication-time selection, dynamic URL, section order, and `nildiff` commands intact. Affected: `.github/workflows/release.yml`, `doc/DEVELOPER_GUIDE.md`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`.
@@ -1706,6 +1708,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
#### 変更
+- **実測に基づくベンチマーク回帰上限** — 単一の緊急上限 200% を、互換履歴の中央値ゲートと、安定性グループ別の warning/failure 上限(10/20%~50/100%)へ置き換えました。ポリシーには、同一定義の `ubuntu-latest` 7 サンプル、観測分散、閾値根拠、定義フィンガープリント入力、最小サンプル数、baseline revision を記録します。Actions summary はすべての pass、warning、failure、warmup、除外した非互換履歴を表示し、制御 fixture で計算と正確な境界を保護します。信頼済み `main` の公開は引き続き自動で、レビュー済みベースラインリセット用に `main` 限定の明示的な手動モードも追加しました。対象: `.github/workflows/benchmark-regression.yml`, `benchmark-regression-policy.json`, `scripts/check_benchmark_regressions.py`, `scripts/tests/test_check_benchmark_regressions.py`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`, `doc/PERFORMANCE_GUIDE.md`, `doc/DEVELOPER_GUIDE.md`, `doc/TESTING_GUIDE.md`。Issue: #225。
+
- **Private vulnerability reporting とセキュリティポリシーの整理** — リポジトリは GitHub private vulnerability reporting を通じて機密報告を受け付けるようになりました。`SECURITY.md` を、サポート対象バージョン、private report への直接リンク、報告内容、対応目標、協調開示、維持した脅威モデルを一読で確認できる、完全な英語節と同等の日本語節へ整理しました。報告リンク、英日ポリシー構造、日本語文書のアンカーを回帰テストで保護します。対象: `SECURITY.md`, `README.md`, `doc/DEVELOPER_GUIDE.md`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`, GitHub リポジトリのセキュリティ設定。Issue: #223.
- **リリースノートの比較リンクを明示的かつ動的に生成** — リリース workflow は公開日時を基準に、非 Draft・非 Prerelease の最新安定 GitHub Release を解決し、そのタグを今回のリリースタグと組み合わせて `Full Changelog` の比較 URL を構築し、NuGet の install/update コマンドを統一されたリリースノートテンプレートへ追加するようになりました。Release の絞り込み、公開日時による選択、動的 URL、セクション順序、`nildiff` コマンドを回帰テストで保護します。対象: `.github/workflows/release.yml`, `doc/DEVELOPER_GUIDE.md`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`。
diff --git a/FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs b/FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs
index db4aff22..5620fd71 100644
--- a/FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs
+++ b/FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs
@@ -627,14 +627,15 @@ public void DotNetWorkflow_BuildsAndSmokeTestsOnMacOs()
}
///
- /// Verifies that the benchmark regression workflow detects performance degradation on PRs.
- /// ベンチマークリグレッションワークフローが PR でパフォーマンス劣化を検知することを検証します。
+ /// Verifies that benchmark regression CI uses evidence-based compatible-history thresholds.
+ /// ベンチマーク回帰 CI が実測に基づく互換履歴の閾値を使用することを検証します。
///
[Fact]
[Trait("Category", "Unit")]
public void BenchmarkRegressionWorkflow_DetectsPerformanceDegradation()
{
var workflow = File.ReadAllText(GetRepositoryFilePath(".github", "workflows", "benchmark-regression.yml"));
+ var policy = File.ReadAllText(GetRepositoryFilePath("benchmark-regression-policy.json"));
Assert.Contains("name: Performance Regression Test", workflow, StringComparison.Ordinal);
Assert.Contains("pull_request:", workflow, StringComparison.Ordinal);
@@ -642,11 +643,25 @@ public void BenchmarkRegressionWorkflow_DetectsPerformanceDegradation()
"benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1",
workflow,
StringComparison.Ordinal);
- Assert.Contains("alert-threshold: '200%'", workflow, StringComparison.Ordinal);
- Assert.Contains("fail-on-alert:", workflow, StringComparison.Ordinal);
Assert.Contains("FolderDiffIL4DotNet.Benchmarks", workflow, StringComparison.Ordinal);
Assert.Contains("--exporters json", workflow, StringComparison.Ordinal);
Assert.Contains("combined-report.json", workflow, StringComparison.Ordinal);
+ Assert.Contains("scripts/check_benchmark_regressions.py", workflow, StringComparison.Ordinal);
+ Assert.Contains("--policy benchmark-regression-policy.json", workflow, StringComparison.Ordinal);
+ Assert.Contains("--baseline-ancestor \"${{ steps.benchmark-base.outputs.sha }}\"", workflow, StringComparison.Ordinal);
+ Assert.Contains("--summary \"$GITHUB_STEP_SUMMARY\"", workflow, StringComparison.Ordinal);
+ Assert.Contains("git show refs/remotes/origin/gh-benchmarks:dev/bench/data.js", workflow, StringComparison.Ordinal);
+ Assert.Contains("publish_baseline:", workflow, StringComparison.Ordinal);
+ Assert.Contains("github.ref == 'refs/heads/main'", workflow, StringComparison.Ordinal);
+ Assert.Contains("github.event.pull_request.base.sha || github.event.before || github.sha", workflow, StringComparison.Ordinal);
+ Assert.DoesNotContain("alert-threshold: '200%'", workflow, StringComparison.Ordinal);
+
+ Assert.Contains("\"sample_count\": 7", policy, StringComparison.Ordinal);
+ Assert.Contains("\"statistic\": \"median of compatible runner-level means\"", policy, StringComparison.Ordinal);
+ Assert.Contains("\"warning_percent\": 10", policy, StringComparison.Ordinal);
+ Assert.Contains("\"failure_percent\": 20", policy, StringComparison.Ordinal);
+ Assert.Contains("\"warning_percent\": 50", policy, StringComparison.Ordinal);
+ Assert.Contains("\"failure_percent\": 100", policy, StringComparison.Ordinal);
}
///
@@ -782,13 +797,16 @@ public void Workflows_RestrictWritePermissionsToTrustedEvents()
StringComparison.Ordinal);
Assert.True(benchmarkPublishStart >= 0);
var benchmarkReadOnlySection = benchmarkWorkflow[..benchmarkPublishStart];
+ var benchmarkPublishSection = benchmarkWorkflow[benchmarkPublishStart..];
Assert.Contains(
- "if: github.event_name == 'push' && github.ref == 'refs/heads/main'",
- benchmarkWorkflow,
+ "(github.event_name == 'push' && github.ref == 'refs/heads/main')",
+ benchmarkPublishSection,
StringComparison.Ordinal);
- Assert.Contains("auto-push: false", benchmarkReadOnlySection, StringComparison.Ordinal);
- Assert.Contains("comment-on-alert: false", benchmarkReadOnlySection, StringComparison.Ordinal);
- Assert.Contains("fail-on-alert: true", benchmarkReadOnlySection, StringComparison.Ordinal);
+ Assert.Contains("github.event_name == 'workflow_dispatch'", benchmarkPublishSection, StringComparison.Ordinal);
+ Assert.Contains("inputs.publish_baseline", benchmarkPublishSection, StringComparison.Ordinal);
+ Assert.DoesNotContain("contents: write", benchmarkReadOnlySection, StringComparison.Ordinal);
+ Assert.DoesNotContain("auto-push: true", benchmarkReadOnlySection, StringComparison.Ordinal);
+ Assert.Contains("scripts/check_benchmark_regressions.py", benchmarkReadOnlySection, StringComparison.Ordinal);
var codeqlAnalyzeStart = codeqlWorkflow.IndexOf(" analyze:", StringComparison.Ordinal);
var codeqlUploadStart = codeqlWorkflow.IndexOf(" upload-results:", StringComparison.Ordinal);
diff --git a/benchmark-regression-policy.json b/benchmark-regression-policy.json
new file mode 100644
index 00000000..a787814a
--- /dev/null
+++ b/benchmark-regression-policy.json
@@ -0,0 +1,84 @@
+{
+ "schema_version": 1,
+ "benchmark_suite": "FolderDiffIL4DotNet Performance",
+ "baseline_revision": 0,
+ "legacy_baseline_revision": 0,
+ "minimum_compatible_samples": 5,
+ "history_window": 10,
+ "benchmark_definition": {
+ "roots": [
+ "FolderDiffIL4DotNet.Benchmarks"
+ ],
+ "files": [
+ "global.json"
+ ]
+ },
+ "evidence": {
+ "source": "gh-benchmarks/dev/bench/data.js",
+ "runner": "ubuntu-latest",
+ "statistic": "median of compatible runner-level means",
+ "sample_count": 7,
+ "first_sample_utc": "2026-07-25T08:40:12.287Z",
+ "last_sample_utc": "2026-07-25T16:48:33.440Z",
+ "compatible_commits": [
+ "c2796a65b9ac24db38b9036020774c4c764b3f0f",
+ "da2bd1f81ee6027112e1cff6989b905e5279480b",
+ "22fec07ddb9dbf5aa7d92a665cb18726b61c5e0c",
+ "aa24e789cafdeacdd9a5bd195a50107cfda55db3",
+ "8163b0b91ff3f62f44a5a0fc892080ef13f5a001",
+ "1f4987e10e43436683664f7fcccf69ab9c26e9ec",
+ "4b32a77390d82468d23363ec8c4121596d64094a"
+ ],
+ "threshold_rule": "Warning is the observed maximum slowdown rounded up to 5 percentage points plus a 5-point guard band; failure is twice the warning threshold."
+ },
+ "groups": [
+ {
+ "name": "folder-io",
+ "patterns": [
+ "FolderDiffIL4DotNet.Benchmarks.FolderDiffBenchmarks.*"
+ ],
+ "observed_max_slowdown_percent": 4.7,
+ "warning_percent": 10,
+ "failure_percent": 20
+ },
+ {
+ "name": "short-path-sanitizers",
+ "patterns": [
+ "FolderDiffIL4DotNet.Benchmarks.ILComparisonBenchmarks.Sanitize_ShortPath",
+ "FolderDiffIL4DotNet.Benchmarks.ILComparisonBenchmarks.Sanitize_UnicodePath"
+ ],
+ "observed_max_slowdown_percent": 13.4,
+ "warning_percent": 20,
+ "failure_percent": 40
+ },
+ {
+ "name": "long-path-sanitizer",
+ "patterns": [
+ "FolderDiffIL4DotNet.Benchmarks.ILComparisonBenchmarks.Sanitize_LongPath"
+ ],
+ "observed_max_slowdown_percent": 30.3,
+ "warning_percent": 40,
+ "failure_percent": 80
+ },
+ {
+ "name": "standard-text-diff",
+ "patterns": [
+ "FolderDiffIL4DotNet.Benchmarks.ILComparisonBenchmarks.TextDiffer_*",
+ "FolderDiffIL4DotNet.Benchmarks.TextDifferBenchmarks.SmallFile_5Changes",
+ "FolderDiffIL4DotNet.Benchmarks.TextDifferBenchmarks.MediumFile_20Changes"
+ ],
+ "observed_max_slowdown_percent": 14.0,
+ "warning_percent": 20,
+ "failure_percent": 40
+ },
+ {
+ "name": "million-line-text-diff",
+ "patterns": [
+ "FolderDiffIL4DotNet.Benchmarks.TextDifferBenchmarks.LargeFile_10Changes"
+ ],
+ "observed_max_slowdown_percent": 42.7,
+ "warning_percent": 50,
+ "failure_percent": 100
+ }
+ ]
+}
diff --git a/doc/DEVELOPER_GUIDE.md b/doc/DEVELOPER_GUIDE.md
index 77551be0..b64a2133 100644
--- a/doc/DEVELOPER_GUIDE.md
+++ b/doc/DEVELOPER_GUIDE.md
@@ -222,7 +222,7 @@ Benchmark classes:
**CI integration:** The `benchmark` job in [`.github/workflows/dotnet.yml`](../.github/workflows/dotnet.yml) runs all benchmarks on `workflow_dispatch` and uploads `BenchmarkDotNet.Artifacts/` as a CI artifact with JSON and GitHub exporters.
-**Regression detection:** The [`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) workflow runs automatically on every PR to `main` and on `push` to `main`. It combines JSON results from all benchmark classes into a single report and uses [`benchmark-action/github-action-benchmark@v1`](https://github.com/benchmark-action/github-action-benchmark) to compare against the stored baseline in the `gh-benchmarks` branch. If any benchmark degrades by more than 50% (alert threshold `150%`), the job fails and a PR comment is posted. On push to `main`, the results are stored as the new baseline.
+**Regression detection:** The [`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) workflow runs automatically on every PR to `main` and on `push` to `main`. It combines all BenchmarkDotNet JSON results, then [`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) compares each mean with the median of compatible trusted history from `gh-benchmarks`. [`benchmark-regression-policy.json`](../benchmark-regression-policy.json) records the seven-sample `ubuntu-latest` evidence, per-stability-group warning/failure limits (`10/20%` through `50/100%`), definition fingerprint inputs, minimum sample count, and baseline revision. Warnings and failures are shown in the Actions summary; only failures block normal runs. [`PERFORMANCE_GUIDE.md`](PERFORMANCE_GUIDE.md#ci-regression-detection) documents the evidence and reset procedure.
### Hash-Based Caching in FileDiffService
@@ -822,9 +822,10 @@ Security automation:
Performance regression detection:
- [`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) runs BenchmarkDotNet on every `pull_request` and `push` to `main`, plus `workflow_dispatch`
-- Combines JSON results from all benchmark classes into a single report and compares against the stored baseline in the `gh-benchmarks` branch using [`benchmark-action/github-action-benchmark@v1`](https://github.com/benchmark-action/github-action-benchmark)
-- Alert threshold is `150%` (50% degradation causes failure); PR comments are posted on regression
-- On push to `main`, results are auto-pushed to `gh-benchmarks` as the new baseline
+- Combines JSON results from all benchmark classes and compares each mean with the median of compatible trusted history using [`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py)
+- Applies the evidence and per-stability-group warning/failure limits in [`benchmark-regression-policy.json`](../benchmark-regression-policy.json); all results are visible in the Actions summary and only failures block normal runs
+- Filters history newer than or unrelated to the intended base, plus incompatible benchmark definitions, SDK selections, and baseline revisions, instead of silently comparing them
+- On successful push to `main`, results are auto-pushed to `gh-benchmarks`; an explicit `publish_baseline` manual run on `main` supports reviewed reset/warmup measurements
- Benchmark artifacts are always uploaded as `BenchmarkResults`
Versioning:
@@ -1199,7 +1200,7 @@ dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks -- --filter *Text
**CI 統合:** [`.github/workflows/dotnet.yml`](../.github/workflows/dotnet.yml) の `benchmark` ジョブは `workflow_dispatch` 時にすべてのベンチマークを実行し、JSON および GitHub エクスポーター付きの `BenchmarkDotNet.Artifacts/` を CI アーティファクトとしてアップロードします。
-**リグレッション検知:** [`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) ワークフローは `main` への PR および `push` のたびに自動実行されます。全ベンチマーククラスの JSON 結果を単一レポートに統合し、[`benchmark-action/github-action-benchmark@v1`](https://github.com/benchmark-action/github-action-benchmark) を使用して `gh-benchmarks` ブランチに保存されたベースラインと比較します。いずれかのベンチマークが 50% 以上劣化した場合(閾値 `150%`)、ジョブが失敗し PR コメントが投稿されます。`main` への push 時には結果が新しいベースラインとして保存されます。
+**リグレッション検知:** [`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) ワークフローは `main` への PR および `push` のたびに自動実行されます。全 BenchmarkDotNet JSON 結果を統合し、[`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) が各 mean を `gh-benchmarks` の互換信頼済み履歴の中央値と比較します。[`benchmark-regression-policy.json`](../benchmark-regression-policy.json) に、`ubuntu-latest` 7 サンプルの根拠、安定性グループ別 warning/failure 上限(`10/20%`~`50/100%`)、定義フィンガープリント入力、最小サンプル数、baseline revision を記録します。warning と failure は Actions summary に表示し、通常実行をブロックするのは failure だけです。根拠とリセット手順は [`PERFORMANCE_GUIDE.md`](PERFORMANCE_GUIDE.md#ci-回帰検出) を参照してください。
### FileDiffService におけるハッシュベースキャッシュ
@@ -1797,9 +1798,10 @@ v* タグ push 時:
パフォーマンスリグレッション検知:
- [`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) は `main` 向け `pull_request` と `push`、および `workflow_dispatch` で BenchmarkDotNet を実行します
-- 全ベンチマーククラスの JSON 結果を単一レポートに統合し、[`benchmark-action/github-action-benchmark@v1`](https://github.com/benchmark-action/github-action-benchmark) を使用して `gh-benchmarks` ブランチに保存されたベースラインと比較します
-- 閾値は `150%`(50% の劣化でジョブ失敗)。リグレッション時に PR コメントを投稿します
-- `main` への push 時には結果が `gh-benchmarks` に新しいベースラインとして自動 push されます
+- 全ベンチマーククラスの JSON 結果を統合し、[`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) が各 mean を互換信頼済み履歴の中央値と比較します
+- [`benchmark-regression-policy.json`](../benchmark-regression-policy.json) の実測根拠と安定性グループ別 warning/failure 上限を適用し、全結果を Actions summary に表示して、通常実行では failure だけをブロッキングにします
+- 意図した base より新しい、または無関係な履歴に加え、非互換なベンチマーク定義、SDK 選択、baseline revision を暗黙に比較せず除外します
+- 成功した `main` push は結果を `gh-benchmarks` へ自動 push し、`main` 上で `publish_baseline` を明示した手動実行はレビュー済み reset/warmup 計測に使用できます
- ベンチマーク成果物は常に `BenchmarkResults` としてアップロードされます
バージョニング:
diff --git a/doc/PERFORMANCE_GUIDE.md b/doc/PERFORMANCE_GUIDE.md
index 2b363379..74669846 100644
--- a/doc/PERFORMANCE_GUIDE.md
+++ b/doc/PERFORMANCE_GUIDE.md
@@ -111,9 +111,21 @@ dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks -- --filter *Fold
### CI Regression Detection
The [`benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) workflow:
-- Stores baseline results in the `gh-benchmarks` branch on push to `main`.
-- Compares PR results against the baseline with a **150% threshold** (50% degradation triggers failure).
-- Posts PR comments when regressions are detected.
+- Stores trusted baseline results in `gh-benchmarks` only after a successful push to `main`, or an explicit `publish_baseline` manual run on `main`.
+- Uses [`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) to compare each current mean with the median of the newest compatible hosted-runner samples. Compatibility requires the history commit to be an ancestor of the intended PR/push base plus the same benchmark-project/global SDK fingerprint and `baseline_revision`; incompatible history is excluded and counted in the Actions summary.
+- Reads warning/failure limits from [`benchmark-regression-policy.json`](../benchmark-regression-policy.json). Warnings remain non-blocking, failures block CI, and every benchmark result and threshold is visible in the Actions summary.
+
+The initial policy is based on seven `ubuntu-latest` runs from 2026-07-25 (`c2796a65` through `4b32a773`) whose benchmark definitions and .NET SDK selection were identical. The median reduces runner outliers. For each stability group, the warning limit is the largest observed slowdown rounded up to 5 percentage points plus a 5-point guard band; the failure limit is twice the warning limit.
+
+| Stability group | Largest observed slowdown | Warning | Failure |
+| --- | ---: | ---: | ---: |
+| Folder enumeration and hashing | 4.7% | 10% | 20% |
+| Short/Unicode path sanitizers | 13.4% | 20% | 40% |
+| Long-path sanitizer | 30.3% | 40% | 80% |
+| Standard text diff | 14.0% | 20% | 40% |
+| Million-line text diff | 42.7% | 50% | 100% |
+
+At least five compatible samples are required; otherwise the affected benchmark is reported as `WARMUP` instead of being compared silently. When a benchmark definition or `global.json` changes, merge the reviewed change, let the `main` push publish the first new baseline, then run the workflow on `main` four more times with `publish_baseline=true`. For an intentional baseline reset without a definition change, increment `baseline_revision` first and follow the same process. Manual publication keeps detected failures visible but non-blocking so an explicitly accepted baseline can be rebuilt. Re-measure hosted-runner variance and update the evidence, thresholds, and changelog when runner images or benchmark behavior materially change.
## Tuning Recommendations
@@ -242,9 +254,21 @@ dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks -- --filter *Fold
### CI 回帰検出
[`benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) ワークフロー:
-- `main` への push 時にベースライン結果を `gh-benchmarks` ブランチに保存。
-- PR 結果をベースラインと **150% 閾値**(50% 劣化で失敗)で比較。
-- 回帰検出時に PR コメントを投稿。
+- `main` への push が成功した場合、または `main` 上で `publish_baseline` を明示した手動実行の場合だけ、信頼済みベースラインを `gh-benchmarks` に保存します。
+- [`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) が、現在の各 mean を互換 hosted-runner サンプルの新しい方から得た中央値と比較します。互換性には、履歴コミットが意図した PR/push の base の祖先であること、ベンチマークプロジェクト/global SDK の同一フィンガープリント、`baseline_revision` を要求し、非互換履歴は除外して Actions summary に件数を表示します。
+- [`benchmark-regression-policy.json`](../benchmark-regression-policy.json) から warning/failure 上限を読み込みます。warning は非ブロッキング、failure は CI をブロックし、すべてのベンチマーク結果と閾値を Actions summary に表示します。
+
+初期ポリシーは、ベンチマーク定義と .NET SDK 選択が同一だった 2026-07-25 の `ubuntu-latest` 7 実行(`c2796a65`~`4b32a773`)に基づきます。runner の外れ値を抑えるため中央値を使用します。各安定性グループの warning は、観測された最大遅化を 5 ポイント単位で切り上げ、さらに 5 ポイントの余裕を加えた値です。failure は warning の 2 倍です。
+
+| 安定性グループ | 観測された最大遅化 | Warning | Failure |
+| --- | ---: | ---: | ---: |
+| フォルダ列挙とハッシュ | 4.7% | 10% | 20% |
+| Short/Unicode パスサニタイザー | 13.4% | 20% | 40% |
+| LongPath サニタイザー | 30.3% | 40% | 80% |
+| 標準テキスト差分 | 14.0% | 20% | 40% |
+| 100 万行テキスト差分 | 42.7% | 50% | 100% |
+
+互換サンプルが 5 件未満の場合、そのベンチマークを暗黙に比較せず `WARMUP` として報告します。ベンチマーク定義または `global.json` を変更した場合は、レビュー済み変更をマージし、`main` push で最初の新ベースラインを公開した後、`main` 上で `publish_baseline=true` を指定してワークフローをさらに 4 回実行します。定義変更なしで意図的にベースラインをリセットする場合は、先に `baseline_revision` を増やして同じ手順を実施します。手動公開モードでも failure は表示しますが、明示的に受け入れたベースラインを再構築できるよう非ブロッキングにします。runner image またはベンチマーク挙動が大きく変わった場合は hosted-runner 分散を再計測し、根拠、閾値、CHANGELOG を更新します。
## チューニング推奨
diff --git a/doc/TESTING_GUIDE.md b/doc/TESTING_GUIDE.md
index 31c2c06d..ead39c04 100644
--- a/doc/TESTING_GUIDE.md
+++ b/doc/TESTING_GUIDE.md
@@ -159,7 +159,7 @@ dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks
dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks -- --filter *TextDiffer*
```
-The `benchmark` CI job (workflow_dispatch only) runs all benchmarks with JSON and GitHub exporters and uploads `BenchmarkDotNet.Artifacts/` as a CI artifact. The `benchmark-regression` workflow ([`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml)) runs automatically on PRs to `main` and detects performance regressions by comparing against stored baselines using [`benchmark-action/github-action-benchmark@v1`](https://github.com/benchmark-action/github-action-benchmark) with a `150%` alert threshold.
+The `benchmark` CI job (workflow_dispatch only) runs all benchmarks with JSON and GitHub exporters and uploads `BenchmarkDotNet.Artifacts/` as a CI artifact. The `benchmark-regression` workflow ([`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml)) runs automatically on PRs to `main`. [`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) uses a compatible-history median and the evidence-based per-group warning/failure limits in [`benchmark-regression-policy.json`](../benchmark-regression-policy.json). Controlled Python fixtures exercise median calculation, exact threshold boundaries, intended-base ancestry, warmup/reset behavior, definition mismatch, and intentional baseline publication; run them with the `python3 -m unittest discover` command above.
CI-parity command (same as GitHub Actions test step):
@@ -411,7 +411,7 @@ dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks
dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks -- --filter *TextDiffer*
```
-`benchmark` CI ジョブ(workflow_dispatch のみ)はすべてのベンチマークを JSON および GitHub エクスポーター付きで実行し、`BenchmarkDotNet.Artifacts/` を CI アーティファクトとしてアップロードします。`benchmark-regression` ワークフロー([`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml))は `main` への PR で自動実行され、[`benchmark-action/github-action-benchmark@v1`](https://github.com/benchmark-action/github-action-benchmark) を使用して保存済みベースラインと比較し、`150%` の閾値でパフォーマンスリグレッションを検知します。
+`benchmark` CI ジョブ(workflow_dispatch のみ)はすべてのベンチマークを JSON および GitHub エクスポーター付きで実行し、`BenchmarkDotNet.Artifacts/` を CI アーティファクトとしてアップロードします。`benchmark-regression` ワークフロー([`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml))は `main` への PR で自動実行されます。[`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) が互換履歴の中央値と [`benchmark-regression-policy.json`](../benchmark-regression-policy.json) の実測ベースなグループ別 warning/failure 上限を使用します。制御された Python fixture で中央値計算、正確な閾値境界、意図した base の祖先判定、warmup/reset、定義不一致、意図的 baseline 公開を検証します。上記の `python3 -m unittest discover` コマンドで実行できます。
CI 同等コマンド(GitHub Actions と同じ test ステップ):
diff --git a/scripts/check_benchmark_regressions.py b/scripts/check_benchmark_regressions.py
new file mode 100644
index 00000000..bde711e1
--- /dev/null
+++ b/scripts/check_benchmark_regressions.py
@@ -0,0 +1,831 @@
+#!/usr/bin/env python3
+"""
+Checks BenchmarkDotNet results against compatible hosted-runner history.
+BenchmarkDotNet の結果を互換性のある hosted runner 履歴と比較します。
+"""
+
+from __future__ import annotations
+
+import argparse
+import fnmatch
+import hashlib
+import json
+import math
+import os
+import re
+import statistics
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+
+BENCHMARK_DATA_PREFIX = "window.BENCHMARK_DATA ="
+COMMIT_PATTERN = re.compile(r"^[0-9a-f]{40}$")
+
+
+class GateError(RuntimeError):
+ """Raised when benchmark inputs or policy are unsafe to evaluate."""
+
+
+@dataclass(frozen=True)
+class ThresholdGroup:
+ """Thresholds shared by benchmarks with comparable observed stability."""
+
+ name: str
+ patterns: tuple[str, ...]
+ warning_percent: float
+ failure_percent: float
+ observed_max_slowdown_percent: float
+
+
+@dataclass(frozen=True)
+class Policy:
+ """Validated benchmark regression policy."""
+
+ benchmark_suite: str
+ baseline_revision: int
+ legacy_baseline_revision: int
+ minimum_compatible_samples: int
+ history_window: int
+ definition_roots: tuple[str, ...]
+ definition_files: tuple[str, ...]
+ groups: tuple[ThresholdGroup, ...]
+
+
+@dataclass(frozen=True)
+class BenchmarkValue:
+ """One benchmark measurement."""
+
+ name: str
+ value: float
+ unit: str
+
+
+@dataclass(frozen=True)
+class HistoryEntry:
+ """One trusted benchmark-history entry."""
+
+ commit: str
+ date: float
+ benchmarks: tuple[BenchmarkValue, ...]
+
+
+@dataclass(frozen=True)
+class Outcome:
+ """Evaluation result for one benchmark."""
+
+ benchmark: str
+ group: str
+ sample_count: int
+ baseline: float | None
+ current: float
+ unit: str
+ change_percent: float | None
+ warning_percent: float
+ failure_percent: float
+ status: str
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(
+ description="Compare BenchmarkDotNet means with compatible trusted history."
+ )
+ parser.add_argument("--current-report", required=True, help="Combined BenchmarkDotNet JSON report.")
+ parser.add_argument("--history-data", required=True, help="github-action-benchmark data.js file.")
+ parser.add_argument("--policy", required=True, help="Evidence-based threshold policy JSON.")
+ parser.add_argument(
+ "--baseline-ancestor",
+ required=True,
+ help="Trusted base commit; history entries must be ancestors of this commit.",
+ )
+ parser.add_argument(
+ "--repository-root",
+ default=".",
+ help="Git repository used to validate benchmark-definition compatibility.",
+ )
+ parser.add_argument(
+ "--summary",
+ help="Markdown summary destination. Defaults to GITHUB_STEP_SUMMARY when available.",
+ )
+ parser.add_argument(
+ "--allow-failure",
+ choices=("true", "false"),
+ default="false",
+ help="Report failures without returning exit 1 during an intentional baseline publication.",
+ )
+ return parser.parse_args()
+
+
+def read_json(path: Path, description: str) -> dict[str, Any]:
+ try:
+ contents = path.read_text(encoding="utf-8")
+ except OSError as error:
+ raise GateError(f"Unable to read {description} '{path}': {error}") from error
+
+ try:
+ value = json.loads(contents)
+ except json.JSONDecodeError as error:
+ raise GateError(f"{description} '{path}' is not valid JSON: {error}") from error
+
+ if not isinstance(value, dict):
+ raise GateError(f"{description} '{path}' must contain a JSON object.")
+ return value
+
+
+def safe_repository_path(value: Any, field: str) -> str:
+ if not isinstance(value, str) or not value:
+ raise GateError(f"{field} must contain non-empty repository-relative paths.")
+
+ path = PurePosixPath(value)
+ if path.is_absolute() or ".." in path.parts or value.startswith("-"):
+ raise GateError(f"{field} contains unsafe repository path '{value}'.")
+ return path.as_posix()
+
+
+def numeric_percent(value: Any, field: str) -> float:
+ if not isinstance(value, (int, float)) or isinstance(value, bool):
+ raise GateError(f"{field} must be numeric.")
+ number = float(value)
+ if not math.isfinite(number) or number < 0:
+ raise GateError(f"{field} must be a finite non-negative number.")
+ return number
+
+
+def positive_integer(value: Any, field: str) -> int:
+ if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
+ raise GateError(f"{field} must be a positive integer.")
+ return value
+
+
+def load_policy(path: Path) -> Policy:
+ payload = read_json(path, "benchmark policy")
+ if payload.get("schema_version") != 1:
+ raise GateError("benchmark policy schema_version must be 1.")
+
+ suite = payload.get("benchmark_suite")
+ if not isinstance(suite, str) or not suite:
+ raise GateError("benchmark_suite must be a non-empty string.")
+
+ baseline_revision = payload.get("baseline_revision")
+ legacy_revision = payload.get("legacy_baseline_revision")
+ if not isinstance(baseline_revision, int) or baseline_revision < 0:
+ raise GateError("baseline_revision must be a non-negative integer.")
+ if not isinstance(legacy_revision, int) or legacy_revision < 0:
+ raise GateError("legacy_baseline_revision must be a non-negative integer.")
+
+ definitions = payload.get("benchmark_definition")
+ if not isinstance(definitions, dict):
+ raise GateError("benchmark_definition must be an object.")
+
+ roots_value = definitions.get("roots")
+ files_value = definitions.get("files")
+ if not isinstance(roots_value, list) or not roots_value:
+ raise GateError("benchmark_definition.roots must be a non-empty array.")
+ if not isinstance(files_value, list):
+ raise GateError("benchmark_definition.files must be an array.")
+
+ roots = tuple(safe_repository_path(value, "benchmark_definition.roots") for value in roots_value)
+ files = tuple(safe_repository_path(value, "benchmark_definition.files") for value in files_value)
+
+ groups_value = payload.get("groups")
+ if not isinstance(groups_value, list) or not groups_value:
+ raise GateError("groups must be a non-empty array.")
+
+ groups: list[ThresholdGroup] = []
+ group_names: set[str] = set()
+ for index, group_value in enumerate(groups_value):
+ field = f"groups[{index}]"
+ if not isinstance(group_value, dict):
+ raise GateError(f"{field} must be an object.")
+
+ name = group_value.get("name")
+ patterns_value = group_value.get("patterns")
+ if not isinstance(name, str) or not name:
+ raise GateError(f"{field}.name must be a non-empty string.")
+ if name in group_names:
+ raise GateError(f"Duplicate threshold group '{name}'.")
+ if not isinstance(patterns_value, list) or not patterns_value:
+ raise GateError(f"{field}.patterns must be a non-empty array.")
+ if not all(isinstance(pattern, str) and pattern for pattern in patterns_value):
+ raise GateError(f"{field}.patterns must contain non-empty strings.")
+
+ warning = numeric_percent(group_value.get("warning_percent"), f"{field}.warning_percent")
+ failure = numeric_percent(group_value.get("failure_percent"), f"{field}.failure_percent")
+ observed = numeric_percent(
+ group_value.get("observed_max_slowdown_percent"),
+ f"{field}.observed_max_slowdown_percent",
+ )
+ if warning <= observed:
+ raise GateError(
+ f"{field}.warning_percent must exceed its observed normal slowdown ({observed:.1f}%)."
+ )
+ if failure <= warning:
+ raise GateError(f"{field}.failure_percent must exceed warning_percent.")
+ if failure >= 200:
+ raise GateError(f"{field}.failure_percent must remain below the retired 200% ceiling.")
+
+ groups.append(
+ ThresholdGroup(
+ name=name,
+ patterns=tuple(patterns_value),
+ warning_percent=warning,
+ failure_percent=failure,
+ observed_max_slowdown_percent=observed,
+ )
+ )
+ group_names.add(name)
+
+ return Policy(
+ benchmark_suite=suite,
+ baseline_revision=baseline_revision,
+ legacy_baseline_revision=legacy_revision,
+ minimum_compatible_samples=positive_integer(
+ payload.get("minimum_compatible_samples"),
+ "minimum_compatible_samples",
+ ),
+ history_window=positive_integer(payload.get("history_window"), "history_window"),
+ definition_roots=roots,
+ definition_files=files,
+ groups=tuple(groups),
+ )
+
+
+def parse_positive_number(value: Any, field: str) -> float:
+ if not isinstance(value, (int, float)) or isinstance(value, bool):
+ raise GateError(f"{field} must be numeric.")
+ number = float(value)
+ if not math.isfinite(number) or number <= 0:
+ raise GateError(f"{field} must be a finite positive number.")
+ return number
+
+
+def load_current_report(path: Path) -> dict[str, BenchmarkValue]:
+ payload = read_json(path, "current benchmark report")
+ benchmarks_value = payload.get("Benchmarks")
+ if not isinstance(benchmarks_value, list) or not benchmarks_value:
+ raise GateError("Current benchmark report must contain a non-empty Benchmarks array.")
+
+ benchmarks: dict[str, BenchmarkValue] = {}
+ for index, item in enumerate(benchmarks_value):
+ if not isinstance(item, dict):
+ raise GateError(f"Benchmarks[{index}] must be an object.")
+ name = item.get("FullName")
+ statistics_value = item.get("Statistics")
+ if not isinstance(name, str) or not name:
+ raise GateError(f"Benchmarks[{index}].FullName must be a non-empty string.")
+ if not isinstance(statistics_value, dict):
+ raise GateError(f"Benchmarks[{index}].Statistics must be an object.")
+ if name in benchmarks:
+ raise GateError(f"Current benchmark report contains duplicate benchmark '{name}'.")
+
+ benchmarks[name] = BenchmarkValue(
+ name=name,
+ value=parse_positive_number(
+ statistics_value.get("Mean"),
+ f"Benchmarks[{index}].Statistics.Mean",
+ ),
+ unit="ns",
+ )
+ return benchmarks
+
+
+def load_history(path: Path, suite: str) -> list[HistoryEntry]:
+ try:
+ contents = path.read_text(encoding="utf-8").strip()
+ except OSError as error:
+ raise GateError(f"Unable to read benchmark history '{path}': {error}") from error
+
+ if not contents.startswith(BENCHMARK_DATA_PREFIX):
+ raise GateError("Benchmark history must start with the expected BENCHMARK_DATA assignment.")
+ json_contents = contents[len(BENCHMARK_DATA_PREFIX) :].strip()
+ if json_contents.endswith(";"):
+ json_contents = json_contents[:-1].rstrip()
+
+ try:
+ payload = json.loads(json_contents)
+ except json.JSONDecodeError as error:
+ raise GateError(f"Benchmark history is not valid JSON data: {error}") from error
+ if not isinstance(payload, dict):
+ raise GateError("Benchmark history payload must be an object.")
+
+ entries_value = payload.get("entries")
+ if not isinstance(entries_value, dict):
+ raise GateError("Benchmark history must contain an entries object.")
+ suite_entries = entries_value.get(suite)
+ if not isinstance(suite_entries, list):
+ raise GateError(f"Benchmark history does not contain suite '{suite}'.")
+
+ entries: list[HistoryEntry] = []
+ for index, entry_value in enumerate(suite_entries):
+ if not isinstance(entry_value, dict):
+ raise GateError(f"History entry {index} must be an object.")
+ if entry_value.get("tool") != "benchmarkdotnet":
+ raise GateError(f"History entry {index} is not a BenchmarkDotNet result.")
+ commit_value = entry_value.get("commit")
+ commit = commit_value.get("id") if isinstance(commit_value, dict) else None
+ if not isinstance(commit, str) or not COMMIT_PATTERN.fullmatch(commit):
+ raise GateError(f"History entry {index} has an invalid full commit SHA.")
+ date = parse_positive_number(entry_value.get("date"), f"History entry {index}.date")
+ benches_value = entry_value.get("benches")
+ if not isinstance(benches_value, list) or not benches_value:
+ raise GateError(f"History entry {index} must contain benchmark values.")
+
+ benchmarks: list[BenchmarkValue] = []
+ names: set[str] = set()
+ for bench_index, benchmark_value in enumerate(benches_value):
+ if not isinstance(benchmark_value, dict):
+ raise GateError(f"History entry {index} benchmark {bench_index} must be an object.")
+ name = benchmark_value.get("name")
+ unit = benchmark_value.get("unit")
+ if not isinstance(name, str) or not name:
+ raise GateError(f"History entry {index} benchmark {bench_index} has no name.")
+ if not isinstance(unit, str) or not unit:
+ raise GateError(f"History entry {index} benchmark {bench_index} has no unit.")
+ if name in names:
+ raise GateError(f"History entry {index} contains duplicate benchmark '{name}'.")
+ benchmarks.append(
+ BenchmarkValue(
+ name=name,
+ value=parse_positive_number(
+ benchmark_value.get("value"),
+ f"History entry {index} benchmark {bench_index}.value",
+ ),
+ unit=unit,
+ )
+ )
+ names.add(name)
+
+ entries.append(HistoryEntry(commit=commit, date=date, benchmarks=tuple(benchmarks)))
+
+ return sorted(entries, key=lambda entry: entry.date)
+
+
+def run_git(repository_root: Path, arguments: list[str], *, allow_failure: bool = False) -> bytes | None:
+ result = subprocess.run(
+ ["git", *arguments],
+ cwd=repository_root,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ check=False,
+ )
+ if result.returncode == 0:
+ return result.stdout
+ if allow_failure:
+ return None
+ stderr = result.stderr.decode("utf-8", errors="replace").strip()
+ raise GateError(f"git {' '.join(arguments)} failed: {stderr or 'unknown error'}")
+
+
+def fingerprint_contents(contents: dict[str, bytes]) -> str:
+ digest = hashlib.sha256()
+ for path in sorted(contents):
+ digest.update(path.encode("utf-8"))
+ digest.update(b"\0")
+ digest.update(contents[path])
+ digest.update(b"\0")
+ return digest.hexdigest()
+
+
+def current_definition_fingerprint(repository_root: Path, policy: Policy) -> str:
+ selectors = [*policy.definition_roots, *policy.definition_files]
+ output = run_git(
+ repository_root,
+ ["ls-files", "-z", "--cached", "--others", "--exclude-standard", "--", *selectors],
+ )
+ assert output is not None
+ paths = sorted(
+ path.decode("utf-8")
+ for path in output.split(b"\0")
+ if path
+ )
+ for required_file in policy.definition_files:
+ if required_file not in paths:
+ raise GateError(f"Benchmark definition file '{required_file}' is missing.")
+ for root in policy.definition_roots:
+ prefix = f"{root.rstrip('/')}/"
+ if not any(path == root or path.startswith(prefix) for path in paths):
+ raise GateError(f"Benchmark definition root '{root}' contains no files.")
+
+ contents: dict[str, bytes] = {}
+ for path in paths:
+ resolved = (repository_root / path).resolve()
+ try:
+ resolved.relative_to(repository_root)
+ except ValueError as error:
+ raise GateError(f"Benchmark definition path '{path}' escapes the repository.") from error
+ try:
+ contents[path] = resolved.read_bytes()
+ except OSError as error:
+ raise GateError(f"Unable to read benchmark definition '{path}': {error}") from error
+ return fingerprint_contents(contents)
+
+
+def historical_definition_fingerprint(
+ repository_root: Path,
+ commit: str,
+ policy: Policy,
+) -> str | None:
+ selectors = [*policy.definition_roots, *policy.definition_files]
+ output = run_git(
+ repository_root,
+ ["ls-tree", "-r", "--name-only", "-z", commit, "--", *selectors],
+ allow_failure=True,
+ )
+ if output is None:
+ return None
+
+ paths = sorted(
+ path.decode("utf-8")
+ for path in output.split(b"\0")
+ if path
+ )
+ if any(required_file not in paths for required_file in policy.definition_files):
+ return None
+ for root in policy.definition_roots:
+ prefix = f"{root.rstrip('/')}/"
+ if not any(path == root or path.startswith(prefix) for path in paths):
+ return None
+
+ contents: dict[str, bytes] = {}
+ for path in paths:
+ content = run_git(
+ repository_root,
+ ["show", f"{commit}:{path}"],
+ allow_failure=True,
+ )
+ if content is None:
+ return None
+ contents[path] = content
+ return fingerprint_contents(contents)
+
+
+def historical_baseline_revision(
+ repository_root: Path,
+ commit: str,
+ policy_relative_path: str,
+ legacy_revision: int,
+) -> int | None:
+ contents = run_git(
+ repository_root,
+ ["show", f"{commit}:{policy_relative_path}"],
+ allow_failure=True,
+ )
+ if contents is None:
+ return legacy_revision
+ try:
+ payload = json.loads(contents.decode("utf-8"))
+ except (UnicodeDecodeError, json.JSONDecodeError):
+ return None
+ revision = payload.get("baseline_revision") if isinstance(payload, dict) else None
+ return revision if isinstance(revision, int) and revision >= 0 else None
+
+
+def compatible_history(
+ repository_root: Path,
+ policy_path: Path,
+ policy: Policy,
+ entries: list[HistoryEntry],
+ baseline_ancestor: str,
+) -> tuple[list[HistoryEntry], dict[str, int], str]:
+ if not COMMIT_PATTERN.fullmatch(baseline_ancestor):
+ raise GateError("baseline ancestor must be a full 40-character lowercase commit SHA.")
+ baseline_exists = run_git(
+ repository_root,
+ ["cat-file", "-e", f"{baseline_ancestor}^{{commit}}"],
+ allow_failure=True,
+ )
+ if baseline_exists is None:
+ raise GateError(f"baseline ancestor commit '{baseline_ancestor}' is unavailable.")
+
+ try:
+ policy_relative_path = policy_path.resolve().relative_to(repository_root).as_posix()
+ except ValueError as error:
+ raise GateError("Benchmark policy must be inside the repository root.") from error
+
+ current_fingerprint = current_definition_fingerprint(repository_root, policy)
+ compatible: list[HistoryEntry] = []
+ exclusions = {
+ "missing_commit": 0,
+ "not_base_ancestor": 0,
+ "definition_mismatch": 0,
+ "revision_mismatch": 0,
+ }
+ fingerprint_cache: dict[str, str | None] = {}
+ revision_cache: dict[str, int | None] = {}
+
+ for entry in entries:
+ commit_exists = run_git(
+ repository_root,
+ ["cat-file", "-e", f"{entry.commit}^{{commit}}"],
+ allow_failure=True,
+ )
+ if commit_exists is None:
+ exclusions["missing_commit"] += 1
+ continue
+
+ is_base_ancestor = run_git(
+ repository_root,
+ ["merge-base", "--is-ancestor", entry.commit, baseline_ancestor],
+ allow_failure=True,
+ )
+ if is_base_ancestor is None:
+ exclusions["not_base_ancestor"] += 1
+ continue
+
+ if entry.commit not in fingerprint_cache:
+ fingerprint_cache[entry.commit] = historical_definition_fingerprint(
+ repository_root,
+ entry.commit,
+ policy,
+ )
+ if fingerprint_cache[entry.commit] != current_fingerprint:
+ exclusions["definition_mismatch"] += 1
+ continue
+
+ if entry.commit not in revision_cache:
+ revision_cache[entry.commit] = historical_baseline_revision(
+ repository_root,
+ entry.commit,
+ policy_relative_path,
+ policy.legacy_baseline_revision,
+ )
+ if revision_cache[entry.commit] != policy.baseline_revision:
+ exclusions["revision_mismatch"] += 1
+ continue
+
+ compatible.append(entry)
+
+ return compatible[-policy.history_window :], exclusions, current_fingerprint
+
+
+def resolve_group(name: str, groups: tuple[ThresholdGroup, ...]) -> ThresholdGroup:
+ matches = [
+ group
+ for group in groups
+ if any(fnmatch.fnmatchcase(name, pattern) for pattern in group.patterns)
+ ]
+ if len(matches) != 1:
+ matched_names = ", ".join(group.name for group in matches) or "none"
+ raise GateError(
+ f"Benchmark '{name}' must match exactly one threshold group; matched: {matched_names}."
+ )
+ return matches[0]
+
+
+def meets_threshold(change_percent: float, threshold_percent: float) -> bool:
+ return change_percent > threshold_percent or math.isclose(
+ change_percent,
+ threshold_percent,
+ rel_tol=1e-12,
+ abs_tol=1e-9,
+ )
+
+
+def evaluate(
+ current: dict[str, BenchmarkValue],
+ entries: list[HistoryEntry],
+ policy: Policy,
+) -> list[Outcome]:
+ outcomes: list[Outcome] = []
+ matched_groups: set[str] = set()
+ for name in sorted(current):
+ measurement = current[name]
+ group = resolve_group(name, policy.groups)
+ matched_groups.add(group.name)
+ samples = [
+ benchmark.value
+ for entry in entries
+ for benchmark in entry.benchmarks
+ if benchmark.name == name and benchmark.unit == measurement.unit
+ ]
+
+ if len(samples) < policy.minimum_compatible_samples:
+ outcomes.append(
+ Outcome(
+ benchmark=name,
+ group=group.name,
+ sample_count=len(samples),
+ baseline=None,
+ current=measurement.value,
+ unit=measurement.unit,
+ change_percent=None,
+ warning_percent=group.warning_percent,
+ failure_percent=group.failure_percent,
+ status="WARMUP",
+ )
+ )
+ continue
+
+ baseline = statistics.median(samples)
+ change_percent = ((measurement.value / baseline) - 1.0) * 100.0
+ if meets_threshold(change_percent, group.failure_percent):
+ status = "FAIL"
+ elif meets_threshold(change_percent, group.warning_percent):
+ status = "WARNING"
+ else:
+ status = "PASS"
+
+ outcomes.append(
+ Outcome(
+ benchmark=name,
+ group=group.name,
+ sample_count=len(samples),
+ baseline=baseline,
+ current=measurement.value,
+ unit=measurement.unit,
+ change_percent=change_percent,
+ warning_percent=group.warning_percent,
+ failure_percent=group.failure_percent,
+ status=status,
+ )
+ )
+
+ unmatched_groups = [group.name for group in policy.groups if group.name not in matched_groups]
+ if unmatched_groups:
+ raise GateError(
+ "Threshold groups matched no current benchmarks: "
+ + ", ".join(unmatched_groups)
+ + "."
+ )
+ return outcomes
+
+
+def markdown_cell(value: str) -> str:
+ return value.replace("\\", "\\\\").replace("|", "\\|").replace("\n", " ")
+
+
+def format_measurement(value: float | None, unit: str) -> str:
+ if value is None:
+ return "—"
+ return f"{value:,.2f} {unit}"
+
+
+def build_summary(
+ outcomes: list[Outcome],
+ compatible_count: int,
+ exclusions: dict[str, int],
+ fingerprint: str,
+ policy: Policy,
+ allow_failure: bool,
+ baseline_ancestor: str,
+) -> str:
+ failures = [outcome for outcome in outcomes if outcome.status == "FAIL"]
+ warnings = [outcome for outcome in outcomes if outcome.status == "WARNING"]
+ warmups = [outcome for outcome in outcomes if outcome.status == "WARMUP"]
+
+ if failures:
+ overall = "FAIL"
+ marker = "❌"
+ elif warnings:
+ overall = "WARNING"
+ marker = "⚠️"
+ elif warmups:
+ overall = "WARMUP"
+ marker = "⚠️"
+ else:
+ overall = "PASS"
+ marker = "✅"
+
+ lines = [
+ "## Performance regression gate",
+ "",
+ f"{marker} **{overall}** — {compatible_count} compatible hosted-runner sample(s); "
+ f"median baseline; revision `{policy.baseline_revision}`.",
+ "",
+ f"- Intended base commit: `{baseline_ancestor}`",
+ f"- Definition fingerprint: `{fingerprint[:16]}`",
+ f"- History window: newest {policy.history_window} compatible samples",
+ f"- Excluded history: definition mismatch {exclusions['definition_mismatch']}, "
+ f"revision mismatch {exclusions['revision_mismatch']}, "
+ f"not an intended-base ancestor {exclusions['not_base_ancestor']}, "
+ f"missing commit {exclusions['missing_commit']}",
+ ]
+ if allow_failure:
+ lines.append(
+ "- Intentional baseline publication mode is enabled: failures remain visible but do not block this run."
+ )
+ if warmups:
+ lines.append(
+ f"- {len(warmups)} benchmark(s) have fewer than "
+ f"{policy.minimum_compatible_samples} compatible samples and are reported as WARMUP."
+ )
+
+ lines.extend(
+ [
+ "",
+ "| Benchmark | Group | Samples | Baseline median | Current mean | Change | Warning | Failure | Result |",
+ "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |",
+ ]
+ )
+ for outcome in outcomes:
+ change = "—" if outcome.change_percent is None else f"{outcome.change_percent:+.1f}%"
+ lines.append(
+ "| "
+ + " | ".join(
+ [
+ markdown_cell(outcome.benchmark),
+ markdown_cell(outcome.group),
+ str(outcome.sample_count),
+ format_measurement(outcome.baseline, outcome.unit),
+ format_measurement(outcome.current, outcome.unit),
+ change,
+ f"{outcome.warning_percent:.0f}%",
+ f"{outcome.failure_percent:.0f}%",
+ outcome.status,
+ ]
+ )
+ + " |"
+ )
+
+ lines.extend(
+ [
+ "",
+ f"Warnings: {len(warnings)}. Failures: {len(failures)}. Warmups: {len(warmups)}.",
+ ]
+ )
+ return "\n".join(lines) + "\n"
+
+
+def build_error_summary(error: GateError) -> str:
+ return "\n".join(
+ [
+ "## Performance regression gate",
+ "",
+ f"❌ **CONFIGURATION ERROR:** {error}",
+ "",
+ ]
+ )
+
+
+def append_summary(path_value: str | None, summary: str) -> None:
+ if not path_value:
+ return
+ path = Path(path_value)
+ try:
+ with path.open("a", encoding="utf-8") as stream:
+ stream.write(summary)
+ except OSError as error:
+ raise GateError(f"Unable to append GitHub job summary '{path}': {error}") from error
+
+
+def emit_annotations(outcomes: list[Outcome]) -> None:
+ for outcome in outcomes:
+ if outcome.status not in {"WARNING", "FAIL"}:
+ continue
+ command = "error" if outcome.status == "FAIL" else "warning"
+ change = outcome.change_percent if outcome.change_percent is not None else 0.0
+ print(
+ f"::{command} title=Benchmark {outcome.status}::{outcome.benchmark} "
+ f"slowed by {change:.1f}% "
+ f"({outcome.warning_percent:.0f}% warning / {outcome.failure_percent:.0f}% failure)."
+ )
+
+
+def main() -> int:
+ args = parse_args()
+ summary_path = args.summary or os.environ.get("GITHUB_STEP_SUMMARY")
+ allow_failure = args.allow_failure == "true"
+
+ try:
+ repository_root = Path(args.repository_root).resolve()
+ policy_path = Path(args.policy).resolve()
+ policy = load_policy(policy_path)
+ current = load_current_report(Path(args.current_report))
+ history = load_history(Path(args.history_data), policy.benchmark_suite)
+ compatible, exclusions, fingerprint = compatible_history(
+ repository_root,
+ policy_path,
+ policy,
+ history,
+ args.baseline_ancestor,
+ )
+ outcomes = evaluate(current, compatible, policy)
+ summary = build_summary(
+ outcomes,
+ len(compatible),
+ exclusions,
+ fingerprint,
+ policy,
+ allow_failure,
+ args.baseline_ancestor,
+ )
+ print(summary, end="")
+ emit_annotations(outcomes)
+ append_summary(summary_path, summary)
+ except GateError as error:
+ summary = build_error_summary(error)
+ print(summary, file=sys.stderr, end="")
+ try:
+ append_summary(summary_path, summary)
+ except GateError as summary_error:
+ print(summary_error, file=sys.stderr)
+ return 2
+
+ has_failure = any(outcome.status == "FAIL" for outcome in outcomes)
+ return 0 if allow_failure or not has_failure else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/tests/test_check_benchmark_regressions.py b/scripts/tests/test_check_benchmark_regressions.py
new file mode 100644
index 00000000..66d109af
--- /dev/null
+++ b/scripts/tests/test_check_benchmark_regressions.py
@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+"""
+Tests the evidence-based benchmark regression gate.
+実測ベースのベンチマーク回帰ゲートをテストします。
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+SCRIPT_PATH = Path(__file__).resolve().parent.parent / "check_benchmark_regressions.py"
+BENCHMARK_NAME = "Example.Benchmarks.Sample"
+
+
+class BenchmarkRegressionGateTests(unittest.TestCase):
+ def setUp(self) -> None:
+ self.temporary_directory = tempfile.TemporaryDirectory()
+ self.root = Path(self.temporary_directory.name)
+ (self.root / "bench").mkdir()
+ (self.root / "bench" / "Benchmark.cs").write_text(
+ "public static class Benchmark {}\n",
+ encoding="utf-8",
+ )
+ self.policy_path = self.root / "benchmark-regression-policy.json"
+ self.write_policy()
+ self.run_git("init")
+ self.run_git("config", "user.name", "Benchmark Test")
+ self.run_git("config", "user.email", "benchmark-test@example.invalid")
+ self.run_git("config", "commit.gpgSign", "false")
+ self.run_git("add", "bench/Benchmark.cs")
+ self.run_git("commit", "-m", "Add benchmark definition")
+ self.commit = self.run_git("rev-parse", "HEAD").stdout.strip()
+
+ def tearDown(self) -> None:
+ self.temporary_directory.cleanup()
+
+ def run_git(self, *arguments: str) -> subprocess.CompletedProcess[str]:
+ result = subprocess.run(
+ ["git", *arguments],
+ cwd=self.root,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ self.assertEqual(0, result.returncode, result.stderr)
+ return result
+
+ def write_policy(self, *, revision: int = 0, include_pattern: bool = True) -> None:
+ pattern = "*" if include_pattern else "Different.*"
+ policy = {
+ "schema_version": 1,
+ "benchmark_suite": "Fixture Suite",
+ "baseline_revision": revision,
+ "legacy_baseline_revision": 0,
+ "minimum_compatible_samples": 3,
+ "history_window": 5,
+ "benchmark_definition": {
+ "roots": ["bench"],
+ "files": [],
+ },
+ "groups": [
+ {
+ "name": "fixture",
+ "patterns": [pattern],
+ "observed_max_slowdown_percent": 10,
+ "warning_percent": 20,
+ "failure_percent": 40,
+ }
+ ],
+ }
+ self.policy_path.write_text(json.dumps(policy), encoding="utf-8")
+
+ def write_current_report(self, value: float) -> Path:
+ path = self.root / "current.json"
+ path.write_text(
+ json.dumps(
+ {
+ "Benchmarks": [
+ {
+ "FullName": BENCHMARK_NAME,
+ "Statistics": {"Mean": value},
+ }
+ ]
+ }
+ ),
+ encoding="utf-8",
+ )
+ return path
+
+ def write_history(self, values: list[float], commit: str | None = None) -> Path:
+ path = self.root / "data.js"
+ entries = []
+ for index, value in enumerate(values):
+ entries.append(
+ {
+ "commit": {"id": commit or self.commit},
+ "date": 1_700_000_000_000 + index,
+ "tool": "benchmarkdotnet",
+ "benches": [
+ {
+ "name": BENCHMARK_NAME,
+ "value": value,
+ "unit": "ns",
+ }
+ ],
+ }
+ )
+ payload = {
+ "entries": {
+ "Fixture Suite": entries,
+ }
+ }
+ path.write_text(
+ f"window.BENCHMARK_DATA = {json.dumps(payload)}",
+ encoding="utf-8",
+ )
+ return path
+
+ def run_gate(
+ self,
+ current_value: float,
+ history_values: list[float],
+ *,
+ allow_failure: bool = False,
+ history_commit: str | None = None,
+ baseline_ancestor: str | None = None,
+ ) -> subprocess.CompletedProcess[str]:
+ current_path = self.write_current_report(current_value)
+ history_path = self.write_history(history_values, history_commit)
+ summary_path = self.root / "summary.md"
+ return subprocess.run(
+ [
+ sys.executable,
+ str(SCRIPT_PATH),
+ "--current-report",
+ str(current_path),
+ "--history-data",
+ str(history_path),
+ "--policy",
+ str(self.policy_path),
+ "--baseline-ancestor",
+ baseline_ancestor or self.commit,
+ "--repository-root",
+ str(self.root),
+ "--summary",
+ str(summary_path),
+ "--allow-failure",
+ str(allow_failure).lower(),
+ ],
+ cwd=self.root,
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+
+ def test_median_threshold_boundaries_produce_pass_warning_and_failure(self) -> None:
+ history = [98, 100, 102]
+
+ passing = self.run_gate(119.9, history)
+ warning = self.run_gate(120, history)
+ failure = self.run_gate(140, history)
+
+ self.assertEqual(0, passing.returncode, passing.stderr)
+ self.assertIn("| PASS |", passing.stdout)
+ self.assertEqual(0, warning.returncode, warning.stderr)
+ self.assertIn("| WARNING |", warning.stdout)
+ self.assertIn("::warning", warning.stdout)
+ self.assertEqual(1, failure.returncode, failure.stderr)
+ self.assertIn("| FAIL |", failure.stdout)
+ self.assertIn("::error", failure.stdout)
+
+ def test_failure_can_be_reported_without_blocking_intentional_baseline_publication(self) -> None:
+ result = self.run_gate(150, [98, 100, 102], allow_failure=True)
+
+ self.assertEqual(0, result.returncode, result.stderr)
+ self.assertIn("Intentional baseline publication mode is enabled", result.stdout)
+ self.assertIn("| FAIL |", result.stdout)
+
+ def test_definition_mismatch_is_visible_and_enters_warmup(self) -> None:
+ (self.root / "bench" / "Benchmark.cs").write_text(
+ "public static class ChangedBenchmark {}\n",
+ encoding="utf-8",
+ )
+
+ result = self.run_gate(500, [98, 100, 102])
+
+ self.assertEqual(0, result.returncode, result.stderr)
+ self.assertIn("definition mismatch 3", result.stdout)
+ self.assertIn("| WARMUP |", result.stdout)
+
+ def test_baseline_revision_change_excludes_old_history(self) -> None:
+ self.write_policy(revision=1)
+
+ result = self.run_gate(500, [98, 100, 102])
+
+ self.assertEqual(0, result.returncode, result.stderr)
+ self.assertIn("revision mismatch 3", result.stdout)
+ self.assertIn("| WARMUP |", result.stdout)
+
+ def test_history_newer_than_intended_base_is_excluded(self) -> None:
+ (self.root / "after-base.txt").write_text("newer\n", encoding="utf-8")
+ self.run_git("add", "after-base.txt")
+ self.run_git("commit", "-m", "Create a commit after the intended base")
+ newer_commit = self.run_git("rev-parse", "HEAD").stdout.strip()
+
+ result = self.run_gate(
+ 500,
+ [98, 100, 102],
+ history_commit=newer_commit,
+ baseline_ancestor=self.commit,
+ )
+
+ self.assertEqual(0, result.returncode, result.stderr)
+ self.assertIn("not an intended-base ancestor 3", result.stdout)
+ self.assertIn("| WARMUP |", result.stdout)
+
+ def test_unassigned_benchmark_fails_closed(self) -> None:
+ self.write_policy(include_pattern=False)
+
+ result = self.run_gate(100, [98, 100, 102])
+
+ self.assertEqual(2, result.returncode)
+ self.assertIn("must match exactly one threshold group", result.stderr)
+
+ def test_policy_rejects_retired_200_percent_failure_ceiling(self) -> None:
+ policy = json.loads(self.policy_path.read_text(encoding="utf-8"))
+ policy["groups"][0]["failure_percent"] = 200
+ self.policy_path.write_text(json.dumps(policy), encoding="utf-8")
+
+ result = self.run_gate(100, [98, 100, 102])
+
+ self.assertEqual(2, result.returncode)
+ self.assertIn("must remain below the retired 200% ceiling", result.stderr)
+
+
+if __name__ == "__main__":
+ unittest.main()