From 45371948f8254f0bf10c3b79a0983a2527e0d9d6 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sun, 26 Jul 2026 02:12:49 +0900 Subject: [PATCH] Add evidence-based benchmark limits --- .github/workflows/benchmark-regression.yml | 67 +- CHANGELOG.md | 4 + .../CiAutomationConfigurationTests.cs | 36 +- benchmark-regression-policy.json | 84 ++ doc/DEVELOPER_GUIDE.md | 18 +- doc/PERFORMANCE_GUIDE.md | 36 +- doc/TESTING_GUIDE.md | 4 +- scripts/check_benchmark_regressions.py | 831 ++++++++++++++++++ .../tests/test_check_benchmark_regressions.py | 243 +++++ 9 files changed, 1279 insertions(+), 44 deletions(-) create mode 100644 benchmark-regression-policy.json create mode 100644 scripts/check_benchmark_regressions.py create mode 100644 scripts/tests/test_check_benchmark_regressions.py diff --git a/.github/workflows/benchmark-regression.yml b/.github/workflows/benchmark-regression.yml index d18bbee6..fefea3c0 100644 --- a/.github/workflows/benchmark-regression.yml +++ b/.github/workflows/benchmark-regression.yml @@ -20,6 +20,12 @@ on: - 'CLAUDE.md' - 'LICENSE' workflow_dispatch: + inputs: + publish_baseline: + description: 'Publish this main-branch run as an intentional trusted baseline' + required: false + default: false + type: boolean permissions: contents: read @@ -84,21 +90,44 @@ jobs: echo "Baseline branch gh-benchmarks does not exist yet. Will be created on first push to main." fi - - name: Detect performance regression + - name: Resolve intended benchmark base + id: benchmark-base + shell: bash + run: | + base_sha="${{ github.event.pull_request.base.sha || github.event.before || github.sha }}" + if [[ ! "$base_sha" =~ ^[0-9a-f]{40}$ ]] || ! git cat-file -e "$base_sha^{commit}"; then + base_sha="$GITHUB_SHA" + fi + echo "sha=$base_sha" >> "$GITHUB_OUTPUT" + + - name: Fetch trusted benchmark history if: steps.check-baseline.outputs.exists == 'true' - uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1 - with: - name: 'FolderDiffIL4DotNet Performance' - tool: 'benchmarkdotnet' - output-file-path: BenchmarkDotNet.Artifacts/results/combined-report.json - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: false - alert-threshold: '200%' - comment-on-alert: false - fail-on-alert: true - summary-always: true - gh-pages-branch: gh-benchmarks - benchmark-data-dir-path: dev/bench + run: | + git fetch --no-tags origin gh-benchmarks:refs/remotes/origin/gh-benchmarks + git show refs/remotes/origin/gh-benchmarks:dev/bench/data.js \ + > BenchmarkDotNet.Artifacts/results/baseline-data.js + + - name: Check evidence-based performance thresholds + if: steps.check-baseline.outputs.exists == 'true' + run: | + python3 scripts/check_benchmark_regressions.py \ + --current-report BenchmarkDotNet.Artifacts/results/combined-report.json \ + --history-data BenchmarkDotNet.Artifacts/results/baseline-data.js \ + --policy benchmark-regression-policy.json \ + --baseline-ancestor "${{ steps.benchmark-base.outputs.sha }}" \ + --repository-root . \ + --summary "$GITHUB_STEP_SUMMARY" \ + --allow-failure "${{ github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.publish_baseline }}" + + - name: Report missing benchmark baseline + if: steps.check-baseline.outputs.exists != 'true' + run: | + { + echo "## Performance regression gate" + echo + echo "⚠️ **WARMUP** — No trusted gh-benchmarks baseline exists yet." + echo "This run remains visible and will seed the baseline only from a trusted main-branch publication." + } >> "$GITHUB_STEP_SUMMARY" - name: Upload benchmark results if: always() @@ -117,7 +146,11 @@ jobs: publish-benchmark-baseline: needs: benchmark-regression - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + if: >- + (github.event_name == 'push' && github.ref == 'refs/heads/main') || + (github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/main' && + inputs.publish_baseline) runs-on: ubuntu-latest permissions: contents: write @@ -161,10 +194,6 @@ jobs: output-file-path: BenchmarkDotNet.Artifacts/results/combined-report.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true - alert-threshold: '200%' - comment-on-alert: false - fail-on-alert: true - summary-always: true gh-pages-branch: gh-benchmarks benchmark-data-dir-path: dev/bench skip-fetch-gh-pages: ${{ steps.check-baseline.outputs.exists != 'true' }} diff --git a/CHANGELOG.md b/CHANGELOG.md index d1fbdebe..736cec13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), #### Changed +- **Evidence-based benchmark regression limits** — Replaced the single 200% emergency ceiling with a compatible-history median gate and per-stability-group warning/failure limits from 10/20% through 50/100%. The policy records seven same-definition `ubuntu-latest` samples, observed variance, threshold rationale, definition fingerprint inputs, sample minimum, and baseline revision. Actions summaries now show every pass, warning, failure, warmup, and excluded incompatible history entry; controlled fixtures protect calculation and exact boundaries. Trusted `main` publication remains automatic, with an explicit main-only manual mode for reviewed baseline resets. Affected: `.github/workflows/benchmark-regression.yml`, `benchmark-regression-policy.json`, `scripts/check_benchmark_regressions.py`, `scripts/tests/test_check_benchmark_regressions.py`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`, `doc/PERFORMANCE_GUIDE.md`, `doc/DEVELOPER_GUIDE.md`, `doc/TESTING_GUIDE.md`. Issue: #225. + - **Private vulnerability reporting and a streamlined security policy** — The repository now accepts confidential reports through GitHub private vulnerability reporting. `SECURITY.md` has one complete English section and one equivalent Japanese section covering supported versions, the direct private-reporting path, report contents, response targets, coordinated disclosure, and the preserved threat model. Regression coverage protects the reporting link, bilingual policy structure, and Japanese documentation anchors. Affected: `SECURITY.md`, `README.md`, `doc/DEVELOPER_GUIDE.md`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`, and the GitHub repository security setting. Issue: #223. - **Release notes now use explicit dynamic comparison links** — The release workflow resolves the latest non-draft, non-prerelease stable GitHub Release by publication time, combines its tag with the current release tag to build the `Full Changelog` comparison URL, and appends the NuGet install/update commands in a consistent release-note template. Regression coverage keeps the release filtering, publication-time selection, dynamic URL, section order, and `nildiff` commands intact. Affected: `.github/workflows/release.yml`, `doc/DEVELOPER_GUIDE.md`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`. @@ -1706,6 +1708,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), #### 変更 +- **実測に基づくベンチマーク回帰上限** — 単一の緊急上限 200% を、互換履歴の中央値ゲートと、安定性グループ別の warning/failure 上限（10/20%～50/100%）へ置き換えました。ポリシーには、同一定義の `ubuntu-latest` 7 サンプル、観測分散、閾値根拠、定義フィンガープリント入力、最小サンプル数、baseline revision を記録します。Actions summary はすべての pass、warning、failure、warmup、除外した非互換履歴を表示し、制御 fixture で計算と正確な境界を保護します。信頼済み `main` の公開は引き続き自動で、レビュー済みベースラインリセット用に `main` 限定の明示的な手動モードも追加しました。対象: `.github/workflows/benchmark-regression.yml`, `benchmark-regression-policy.json`, `scripts/check_benchmark_regressions.py`, `scripts/tests/test_check_benchmark_regressions.py`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`, `doc/PERFORMANCE_GUIDE.md`, `doc/DEVELOPER_GUIDE.md`, `doc/TESTING_GUIDE.md`。Issue: #225。 + - **Private vulnerability reporting とセキュリティポリシーの整理** — リポジトリは GitHub private vulnerability reporting を通じて機密報告を受け付けるようになりました。`SECURITY.md` を、サポート対象バージョン、private report への直接リンク、報告内容、対応目標、協調開示、維持した脅威モデルを一読で確認できる、完全な英語節と同等の日本語節へ整理しました。報告リンク、英日ポリシー構造、日本語文書のアンカーを回帰テストで保護します。対象: `SECURITY.md`, `README.md`, `doc/DEVELOPER_GUIDE.md`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`, GitHub リポジトリのセキュリティ設定。Issue: #223. - **リリースノートの比較リンクを明示的かつ動的に生成** — リリース workflow は公開日時を基準に、非 Draft・非 Prerelease の最新安定 GitHub Release を解決し、そのタグを今回のリリースタグと組み合わせて `Full Changelog` の比較 URL を構築し、NuGet の install/update コマンドを統一されたリリースノートテンプレートへ追加するようになりました。Release の絞り込み、公開日時による選択、動的 URL、セクション順序、`nildiff` コマンドを回帰テストで保護します。対象: `.github/workflows/release.yml`, `doc/DEVELOPER_GUIDE.md`, `FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs`。 diff --git a/FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs b/FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs index db4aff22..5620fd71 100644 --- a/FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs +++ b/FolderDiffIL4DotNet.Tests/Architecture/CiAutomationConfigurationTests.cs @@ -627,14 +627,15 @@ public void DotNetWorkflow_BuildsAndSmokeTestsOnMacOs() } /// - /// Verifies that the benchmark regression workflow detects performance degradation on PRs. - /// ベンチマークリグレッションワークフローが PR でパフォーマンス劣化を検知することを検証します。 + /// Verifies that benchmark regression CI uses evidence-based compatible-history thresholds. + /// ベンチマーク回帰 CI が実測に基づく互換履歴の閾値を使用することを検証します。 /// [Fact] [Trait("Category", "Unit")] public void BenchmarkRegressionWorkflow_DetectsPerformanceDegradation() { var workflow = File.ReadAllText(GetRepositoryFilePath(".github", "workflows", "benchmark-regression.yml")); + var policy = File.ReadAllText(GetRepositoryFilePath("benchmark-regression-policy.json")); Assert.Contains("name: Performance Regression Test", workflow, StringComparison.Ordinal); Assert.Contains("pull_request:", workflow, StringComparison.Ordinal); @@ -642,11 +643,25 @@ public void BenchmarkRegressionWorkflow_DetectsPerformanceDegradation() "benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1", workflow, StringComparison.Ordinal); - Assert.Contains("alert-threshold: '200%'", workflow, StringComparison.Ordinal); - Assert.Contains("fail-on-alert:", workflow, StringComparison.Ordinal); Assert.Contains("FolderDiffIL4DotNet.Benchmarks", workflow, StringComparison.Ordinal); Assert.Contains("--exporters json", workflow, StringComparison.Ordinal); Assert.Contains("combined-report.json", workflow, StringComparison.Ordinal); + Assert.Contains("scripts/check_benchmark_regressions.py", workflow, StringComparison.Ordinal); + Assert.Contains("--policy benchmark-regression-policy.json", workflow, StringComparison.Ordinal); + Assert.Contains("--baseline-ancestor \"${{ steps.benchmark-base.outputs.sha }}\"", workflow, StringComparison.Ordinal); + Assert.Contains("--summary \"$GITHUB_STEP_SUMMARY\"", workflow, StringComparison.Ordinal); + Assert.Contains("git show refs/remotes/origin/gh-benchmarks:dev/bench/data.js", workflow, StringComparison.Ordinal); + Assert.Contains("publish_baseline:", workflow, StringComparison.Ordinal); + Assert.Contains("github.ref == 'refs/heads/main'", workflow, StringComparison.Ordinal); + Assert.Contains("github.event.pull_request.base.sha || github.event.before || github.sha", workflow, StringComparison.Ordinal); + Assert.DoesNotContain("alert-threshold: '200%'", workflow, StringComparison.Ordinal); + + Assert.Contains("\"sample_count\": 7", policy, StringComparison.Ordinal); + Assert.Contains("\"statistic\": \"median of compatible runner-level means\"", policy, StringComparison.Ordinal); + Assert.Contains("\"warning_percent\": 10", policy, StringComparison.Ordinal); + Assert.Contains("\"failure_percent\": 20", policy, StringComparison.Ordinal); + Assert.Contains("\"warning_percent\": 50", policy, StringComparison.Ordinal); + Assert.Contains("\"failure_percent\": 100", policy, StringComparison.Ordinal); } /// @@ -782,13 +797,16 @@ public void Workflows_RestrictWritePermissionsToTrustedEvents() StringComparison.Ordinal); Assert.True(benchmarkPublishStart >= 0); var benchmarkReadOnlySection = benchmarkWorkflow[..benchmarkPublishStart]; + var benchmarkPublishSection = benchmarkWorkflow[benchmarkPublishStart..]; Assert.Contains( - "if: github.event_name == 'push' && github.ref == 'refs/heads/main'", - benchmarkWorkflow, + "(github.event_name == 'push' && github.ref == 'refs/heads/main')", + benchmarkPublishSection, StringComparison.Ordinal); - Assert.Contains("auto-push: false", benchmarkReadOnlySection, StringComparison.Ordinal); - Assert.Contains("comment-on-alert: false", benchmarkReadOnlySection, StringComparison.Ordinal); - Assert.Contains("fail-on-alert: true", benchmarkReadOnlySection, StringComparison.Ordinal); + Assert.Contains("github.event_name == 'workflow_dispatch'", benchmarkPublishSection, StringComparison.Ordinal); + Assert.Contains("inputs.publish_baseline", benchmarkPublishSection, StringComparison.Ordinal); + Assert.DoesNotContain("contents: write", benchmarkReadOnlySection, StringComparison.Ordinal); + Assert.DoesNotContain("auto-push: true", benchmarkReadOnlySection, StringComparison.Ordinal); + Assert.Contains("scripts/check_benchmark_regressions.py", benchmarkReadOnlySection, StringComparison.Ordinal); var codeqlAnalyzeStart = codeqlWorkflow.IndexOf(" analyze:", StringComparison.Ordinal); var codeqlUploadStart = codeqlWorkflow.IndexOf(" upload-results:", StringComparison.Ordinal); diff --git a/benchmark-regression-policy.json b/benchmark-regression-policy.json new file mode 100644 index 00000000..a787814a --- /dev/null +++ b/benchmark-regression-policy.json @@ -0,0 +1,84 @@ +{ + "schema_version": 1, + "benchmark_suite": "FolderDiffIL4DotNet Performance", + "baseline_revision": 0, + "legacy_baseline_revision": 0, + "minimum_compatible_samples": 5, + "history_window": 10, + "benchmark_definition": { + "roots": [ + "FolderDiffIL4DotNet.Benchmarks" + ], + "files": [ + "global.json" + ] + }, + "evidence": { + "source": "gh-benchmarks/dev/bench/data.js", + "runner": "ubuntu-latest", + "statistic": "median of compatible runner-level means", + "sample_count": 7, + "first_sample_utc": "2026-07-25T08:40:12.287Z", + "last_sample_utc": "2026-07-25T16:48:33.440Z", + "compatible_commits": [ + "c2796a65b9ac24db38b9036020774c4c764b3f0f", + "da2bd1f81ee6027112e1cff6989b905e5279480b", + "22fec07ddb9dbf5aa7d92a665cb18726b61c5e0c", + "aa24e789cafdeacdd9a5bd195a50107cfda55db3", + "8163b0b91ff3f62f44a5a0fc892080ef13f5a001", + "1f4987e10e43436683664f7fcccf69ab9c26e9ec", + "4b32a77390d82468d23363ec8c4121596d64094a" + ], + "threshold_rule": "Warning is the observed maximum slowdown rounded up to 5 percentage points plus a 5-point guard band; failure is twice the warning threshold." + }, + "groups": [ + { + "name": "folder-io", + "patterns": [ + "FolderDiffIL4DotNet.Benchmarks.FolderDiffBenchmarks.*" + ], + "observed_max_slowdown_percent": 4.7, + "warning_percent": 10, + "failure_percent": 20 + }, + { + "name": "short-path-sanitizers", + "patterns": [ + "FolderDiffIL4DotNet.Benchmarks.ILComparisonBenchmarks.Sanitize_ShortPath", + "FolderDiffIL4DotNet.Benchmarks.ILComparisonBenchmarks.Sanitize_UnicodePath" + ], + "observed_max_slowdown_percent": 13.4, + "warning_percent": 20, + "failure_percent": 40 + }, + { + "name": "long-path-sanitizer", + "patterns": [ + "FolderDiffIL4DotNet.Benchmarks.ILComparisonBenchmarks.Sanitize_LongPath" + ], + "observed_max_slowdown_percent": 30.3, + "warning_percent": 40, + "failure_percent": 80 + }, + { + "name": "standard-text-diff", + "patterns": [ + "FolderDiffIL4DotNet.Benchmarks.ILComparisonBenchmarks.TextDiffer_*", + "FolderDiffIL4DotNet.Benchmarks.TextDifferBenchmarks.SmallFile_5Changes", + "FolderDiffIL4DotNet.Benchmarks.TextDifferBenchmarks.MediumFile_20Changes" + ], + "observed_max_slowdown_percent": 14.0, + "warning_percent": 20, + "failure_percent": 40 + }, + { + "name": "million-line-text-diff", + "patterns": [ + "FolderDiffIL4DotNet.Benchmarks.TextDifferBenchmarks.LargeFile_10Changes" + ], + "observed_max_slowdown_percent": 42.7, + "warning_percent": 50, + "failure_percent": 100 + } + ] +} diff --git a/doc/DEVELOPER_GUIDE.md b/doc/DEVELOPER_GUIDE.md index 77551be0..b64a2133 100644 --- a/doc/DEVELOPER_GUIDE.md +++ b/doc/DEVELOPER_GUIDE.md @@ -222,7 +222,7 @@ Benchmark classes: **CI integration:** The `benchmark` job in [`.github/workflows/dotnet.yml`](../.github/workflows/dotnet.yml) runs all benchmarks on `workflow_dispatch` and uploads `BenchmarkDotNet.Artifacts/` as a CI artifact with JSON and GitHub exporters. -**Regression detection:** The [`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) workflow runs automatically on every PR to `main` and on `push` to `main`. It combines JSON results from all benchmark classes into a single report and uses [`benchmark-action/github-action-benchmark@v1`](https://github.com/benchmark-action/github-action-benchmark) to compare against the stored baseline in the `gh-benchmarks` branch. If any benchmark degrades by more than 50% (alert threshold `150%`), the job fails and a PR comment is posted. On push to `main`, the results are stored as the new baseline. +**Regression detection:** The [`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) workflow runs automatically on every PR to `main` and on `push` to `main`. It combines all BenchmarkDotNet JSON results, then [`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) compares each mean with the median of compatible trusted history from `gh-benchmarks`. [`benchmark-regression-policy.json`](../benchmark-regression-policy.json) records the seven-sample `ubuntu-latest` evidence, per-stability-group warning/failure limits (`10/20%` through `50/100%`), definition fingerprint inputs, minimum sample count, and baseline revision. Warnings and failures are shown in the Actions summary; only failures block normal runs. [`PERFORMANCE_GUIDE.md`](PERFORMANCE_GUIDE.md#ci-regression-detection) documents the evidence and reset procedure. ### Hash-Based Caching in FileDiffService @@ -822,9 +822,10 @@ Security automation: Performance regression detection: - [`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) runs BenchmarkDotNet on every `pull_request` and `push` to `main`, plus `workflow_dispatch` -- Combines JSON results from all benchmark classes into a single report and compares against the stored baseline in the `gh-benchmarks` branch using [`benchmark-action/github-action-benchmark@v1`](https://github.com/benchmark-action/github-action-benchmark) -- Alert threshold is `150%` (50% degradation causes failure); PR comments are posted on regression -- On push to `main`, results are auto-pushed to `gh-benchmarks` as the new baseline +- Combines JSON results from all benchmark classes and compares each mean with the median of compatible trusted history using [`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) +- Applies the evidence and per-stability-group warning/failure limits in [`benchmark-regression-policy.json`](../benchmark-regression-policy.json); all results are visible in the Actions summary and only failures block normal runs +- Filters history newer than or unrelated to the intended base, plus incompatible benchmark definitions, SDK selections, and baseline revisions, instead of silently comparing them +- On successful push to `main`, results are auto-pushed to `gh-benchmarks`; an explicit `publish_baseline` manual run on `main` supports reviewed reset/warmup measurements - Benchmark artifacts are always uploaded as `BenchmarkResults` Versioning: @@ -1199,7 +1200,7 @@ dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks -- --filter *Text **CI 統合:** [`.github/workflows/dotnet.yml`](../.github/workflows/dotnet.yml) の `benchmark` ジョブは `workflow_dispatch` 時にすべてのベンチマークを実行し、JSON および GitHub エクスポーター付きの `BenchmarkDotNet.Artifacts/` を CI アーティファクトとしてアップロードします。 -**リグレッション検知:** [`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) ワークフローは `main` への PR および `push` のたびに自動実行されます。全ベンチマーククラスの JSON 結果を単一レポートに統合し、[`benchmark-action/github-action-benchmark@v1`](https://github.com/benchmark-action/github-action-benchmark) を使用して `gh-benchmarks` ブランチに保存されたベースラインと比較します。いずれかのベンチマークが 50% 以上劣化した場合（閾値 `150%`）、ジョブが失敗し PR コメントが投稿されます。`main` への push 時には結果が新しいベースラインとして保存されます。 +**リグレッション検知:** [`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) ワークフローは `main` への PR および `push` のたびに自動実行されます。全 BenchmarkDotNet JSON 結果を統合し、[`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) が各 mean を `gh-benchmarks` の互換信頼済み履歴の中央値と比較します。[`benchmark-regression-policy.json`](../benchmark-regression-policy.json) に、`ubuntu-latest` 7 サンプルの根拠、安定性グループ別 warning/failure 上限（`10/20%`～`50/100%`）、定義フィンガープリント入力、最小サンプル数、baseline revision を記録します。warning と failure は Actions summary に表示し、通常実行をブロックするのは failure だけです。根拠とリセット手順は [`PERFORMANCE_GUIDE.md`](PERFORMANCE_GUIDE.md#ci-回帰検出) を参照してください。 ### FileDiffService におけるハッシュベースキャッシュ @@ -1797,9 +1798,10 @@ v* タグ push 時: パフォーマンスリグレッション検知: - [`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) は `main` 向け `pull_request` と `push`、および `workflow_dispatch` で BenchmarkDotNet を実行します -- 全ベンチマーククラスの JSON 結果を単一レポートに統合し、[`benchmark-action/github-action-benchmark@v1`](https://github.com/benchmark-action/github-action-benchmark) を使用して `gh-benchmarks` ブランチに保存されたベースラインと比較します -- 閾値は `150%`（50% の劣化でジョブ失敗）。リグレッション時に PR コメントを投稿します -- `main` への push 時には結果が `gh-benchmarks` に新しいベースラインとして自動 push されます +- 全ベンチマーククラスの JSON 結果を統合し、[`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) が各 mean を互換信頼済み履歴の中央値と比較します +- [`benchmark-regression-policy.json`](../benchmark-regression-policy.json) の実測根拠と安定性グループ別 warning/failure 上限を適用し、全結果を Actions summary に表示して、通常実行では failure だけをブロッキングにします +- 意図した base より新しい、または無関係な履歴に加え、非互換なベンチマーク定義、SDK 選択、baseline revision を暗黙に比較せず除外します +- 成功した `main` push は結果を `gh-benchmarks` へ自動 push し、`main` 上で `publish_baseline` を明示した手動実行はレビュー済み reset/warmup 計測に使用できます - ベンチマーク成果物は常に `BenchmarkResults` としてアップロードされます バージョニング: diff --git a/doc/PERFORMANCE_GUIDE.md b/doc/PERFORMANCE_GUIDE.md index 2b363379..74669846 100644 --- a/doc/PERFORMANCE_GUIDE.md +++ b/doc/PERFORMANCE_GUIDE.md @@ -111,9 +111,21 @@ dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks -- --filter *Fold ### CI Regression Detection The [`benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) workflow: -- Stores baseline results in the `gh-benchmarks` branch on push to `main`. -- Compares PR results against the baseline with a **150% threshold** (50% degradation triggers failure). -- Posts PR comments when regressions are detected. +- Stores trusted baseline results in `gh-benchmarks` only after a successful push to `main`, or an explicit `publish_baseline` manual run on `main`. +- Uses [`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) to compare each current mean with the median of the newest compatible hosted-runner samples. Compatibility requires the history commit to be an ancestor of the intended PR/push base plus the same benchmark-project/global SDK fingerprint and `baseline_revision`; incompatible history is excluded and counted in the Actions summary. +- Reads warning/failure limits from [`benchmark-regression-policy.json`](../benchmark-regression-policy.json). Warnings remain non-blocking, failures block CI, and every benchmark result and threshold is visible in the Actions summary. + +The initial policy is based on seven `ubuntu-latest` runs from 2026-07-25 (`c2796a65` through `4b32a773`) whose benchmark definitions and .NET SDK selection were identical. The median reduces runner outliers. For each stability group, the warning limit is the largest observed slowdown rounded up to 5 percentage points plus a 5-point guard band; the failure limit is twice the warning limit. + +| Stability group | Largest observed slowdown | Warning | Failure | +| --- | ---: | ---: | ---: | +| Folder enumeration and hashing | 4.7% | 10% | 20% | +| Short/Unicode path sanitizers | 13.4% | 20% | 40% | +| Long-path sanitizer | 30.3% | 40% | 80% | +| Standard text diff | 14.0% | 20% | 40% | +| Million-line text diff | 42.7% | 50% | 100% | + +At least five compatible samples are required; otherwise the affected benchmark is reported as `WARMUP` instead of being compared silently. When a benchmark definition or `global.json` changes, merge the reviewed change, let the `main` push publish the first new baseline, then run the workflow on `main` four more times with `publish_baseline=true`. For an intentional baseline reset without a definition change, increment `baseline_revision` first and follow the same process. Manual publication keeps detected failures visible but non-blocking so an explicitly accepted baseline can be rebuilt. Re-measure hosted-runner variance and update the evidence, thresholds, and changelog when runner images or benchmark behavior materially change. ## Tuning Recommendations @@ -242,9 +254,21 @@ dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks -- --filter *Fold ### CI 回帰検出 [`benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml) ワークフロー: -- `main` への push 時にベースライン結果を `gh-benchmarks` ブランチに保存。 -- PR 結果をベースラインと **150% 閾値**（50% 劣化で失敗）で比較。 -- 回帰検出時に PR コメントを投稿。 +- `main` への push が成功した場合、または `main` 上で `publish_baseline` を明示した手動実行の場合だけ、信頼済みベースラインを `gh-benchmarks` に保存します。 +- [`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) が、現在の各 mean を互換 hosted-runner サンプルの新しい方から得た中央値と比較します。互換性には、履歴コミットが意図した PR/push の base の祖先であること、ベンチマークプロジェクト/global SDK の同一フィンガープリント、`baseline_revision` を要求し、非互換履歴は除外して Actions summary に件数を表示します。 +- [`benchmark-regression-policy.json`](../benchmark-regression-policy.json) から warning/failure 上限を読み込みます。warning は非ブロッキング、failure は CI をブロックし、すべてのベンチマーク結果と閾値を Actions summary に表示します。 + +初期ポリシーは、ベンチマーク定義と .NET SDK 選択が同一だった 2026-07-25 の `ubuntu-latest` 7 実行（`c2796a65`～`4b32a773`）に基づきます。runner の外れ値を抑えるため中央値を使用します。各安定性グループの warning は、観測された最大遅化を 5 ポイント単位で切り上げ、さらに 5 ポイントの余裕を加えた値です。failure は warning の 2 倍です。 + +| 安定性グループ | 観測された最大遅化 | Warning | Failure | +| --- | ---: | ---: | ---: | +| フォルダ列挙とハッシュ | 4.7% | 10% | 20% | +| Short/Unicode パスサニタイザー | 13.4% | 20% | 40% | +| LongPath サニタイザー | 30.3% | 40% | 80% | +| 標準テキスト差分 | 14.0% | 20% | 40% | +| 100 万行テキスト差分 | 42.7% | 50% | 100% | + +互換サンプルが 5 件未満の場合、そのベンチマークを暗黙に比較せず `WARMUP` として報告します。ベンチマーク定義または `global.json` を変更した場合は、レビュー済み変更をマージし、`main` push で最初の新ベースラインを公開した後、`main` 上で `publish_baseline=true` を指定してワークフローをさらに 4 回実行します。定義変更なしで意図的にベースラインをリセットする場合は、先に `baseline_revision` を増やして同じ手順を実施します。手動公開モードでも failure は表示しますが、明示的に受け入れたベースラインを再構築できるよう非ブロッキングにします。runner image またはベンチマーク挙動が大きく変わった場合は hosted-runner 分散を再計測し、根拠、閾値、CHANGELOG を更新します。 ## チューニング推奨 diff --git a/doc/TESTING_GUIDE.md b/doc/TESTING_GUIDE.md index 31c2c06d..ead39c04 100644 --- a/doc/TESTING_GUIDE.md +++ b/doc/TESTING_GUIDE.md @@ -159,7 +159,7 @@ dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks -- --filter *TextDiffer* ``` -The `benchmark` CI job (workflow_dispatch only) runs all benchmarks with JSON and GitHub exporters and uploads `BenchmarkDotNet.Artifacts/` as a CI artifact. The `benchmark-regression` workflow ([`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml)) runs automatically on PRs to `main` and detects performance regressions by comparing against stored baselines using [`benchmark-action/github-action-benchmark@v1`](https://github.com/benchmark-action/github-action-benchmark) with a `150%` alert threshold. +The `benchmark` CI job (workflow_dispatch only) runs all benchmarks with JSON and GitHub exporters and uploads `BenchmarkDotNet.Artifacts/` as a CI artifact. The `benchmark-regression` workflow ([`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml)) runs automatically on PRs to `main`. [`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) uses a compatible-history median and the evidence-based per-group warning/failure limits in [`benchmark-regression-policy.json`](../benchmark-regression-policy.json). Controlled Python fixtures exercise median calculation, exact threshold boundaries, intended-base ancestry, warmup/reset behavior, definition mismatch, and intentional baseline publication; run them with the `python3 -m unittest discover` command above. CI-parity command (same as GitHub Actions test step): @@ -411,7 +411,7 @@ dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks dotnet run -c Release --project FolderDiffIL4DotNet.Benchmarks -- --filter *TextDiffer* ``` -`benchmark` CI ジョブ（workflow_dispatch のみ）はすべてのベンチマークを JSON および GitHub エクスポーター付きで実行し、`BenchmarkDotNet.Artifacts/` を CI アーティファクトとしてアップロードします。`benchmark-regression` ワークフロー（[`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml)）は `main` への PR で自動実行され、[`benchmark-action/github-action-benchmark@v1`](https://github.com/benchmark-action/github-action-benchmark) を使用して保存済みベースラインと比較し、`150%` の閾値でパフォーマンスリグレッションを検知します。 +`benchmark` CI ジョブ（workflow_dispatch のみ）はすべてのベンチマークを JSON および GitHub エクスポーター付きで実行し、`BenchmarkDotNet.Artifacts/` を CI アーティファクトとしてアップロードします。`benchmark-regression` ワークフロー（[`.github/workflows/benchmark-regression.yml`](../.github/workflows/benchmark-regression.yml)）は `main` への PR で自動実行されます。[`scripts/check_benchmark_regressions.py`](../scripts/check_benchmark_regressions.py) が互換履歴の中央値と [`benchmark-regression-policy.json`](../benchmark-regression-policy.json) の実測ベースなグループ別 warning/failure 上限を使用します。制御された Python fixture で中央値計算、正確な閾値境界、意図した base の祖先判定、warmup/reset、定義不一致、意図的 baseline 公開を検証します。上記の `python3 -m unittest discover` コマンドで実行できます。 CI 同等コマンド（GitHub Actions と同じ test ステップ）: diff --git a/scripts/check_benchmark_regressions.py b/scripts/check_benchmark_regressions.py new file mode 100644 index 00000000..bde711e1 --- /dev/null +++ b/scripts/check_benchmark_regressions.py @@ -0,0 +1,831 @@ +#!/usr/bin/env python3 +""" +Checks BenchmarkDotNet results against compatible hosted-runner history. +BenchmarkDotNet の結果を互換性のある hosted runner 履歴と比較します。 +""" + +from __future__ import annotations + +import argparse +import fnmatch +import hashlib +import json +import math +import os +import re +import statistics +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + + +BENCHMARK_DATA_PREFIX = "window.BENCHMARK_DATA =" +COMMIT_PATTERN = re.compile(r"^[0-9a-f]{40}$") + + +class GateError(RuntimeError): + """Raised when benchmark inputs or policy are unsafe to evaluate.""" + + +@dataclass(frozen=True) +class ThresholdGroup: + """Thresholds shared by benchmarks with comparable observed stability.""" + + name: str + patterns: tuple[str, ...] + warning_percent: float + failure_percent: float + observed_max_slowdown_percent: float + + +@dataclass(frozen=True) +class Policy: + """Validated benchmark regression policy.""" + + benchmark_suite: str + baseline_revision: int + legacy_baseline_revision: int + minimum_compatible_samples: int + history_window: int + definition_roots: tuple[str, ...] + definition_files: tuple[str, ...] + groups: tuple[ThresholdGroup, ...] + + +@dataclass(frozen=True) +class BenchmarkValue: + """One benchmark measurement.""" + + name: str + value: float + unit: str + + +@dataclass(frozen=True) +class HistoryEntry: + """One trusted benchmark-history entry.""" + + commit: str + date: float + benchmarks: tuple[BenchmarkValue, ...] + + +@dataclass(frozen=True) +class Outcome: + """Evaluation result for one benchmark.""" + + benchmark: str + group: str + sample_count: int + baseline: float | None + current: float + unit: str + change_percent: float | None + warning_percent: float + failure_percent: float + status: str + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Compare BenchmarkDotNet means with compatible trusted history." + ) + parser.add_argument("--current-report", required=True, help="Combined BenchmarkDotNet JSON report.") + parser.add_argument("--history-data", required=True, help="github-action-benchmark data.js file.") + parser.add_argument("--policy", required=True, help="Evidence-based threshold policy JSON.") + parser.add_argument( + "--baseline-ancestor", + required=True, + help="Trusted base commit; history entries must be ancestors of this commit.", + ) + parser.add_argument( + "--repository-root", + default=".", + help="Git repository used to validate benchmark-definition compatibility.", + ) + parser.add_argument( + "--summary", + help="Markdown summary destination. Defaults to GITHUB_STEP_SUMMARY when available.", + ) + parser.add_argument( + "--allow-failure", + choices=("true", "false"), + default="false", + help="Report failures without returning exit 1 during an intentional baseline publication.", + ) + return parser.parse_args() + + +def read_json(path: Path, description: str) -> dict[str, Any]: + try: + contents = path.read_text(encoding="utf-8") + except OSError as error: + raise GateError(f"Unable to read {description} '{path}': {error}") from error + + try: + value = json.loads(contents) + except json.JSONDecodeError as error: + raise GateError(f"{description} '{path}' is not valid JSON: {error}") from error + + if not isinstance(value, dict): + raise GateError(f"{description} '{path}' must contain a JSON object.") + return value + + +def safe_repository_path(value: Any, field: str) -> str: + if not isinstance(value, str) or not value: + raise GateError(f"{field} must contain non-empty repository-relative paths.") + + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts or value.startswith("-"): + raise GateError(f"{field} contains unsafe repository path '{value}'.") + return path.as_posix() + + +def numeric_percent(value: Any, field: str) -> float: + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise GateError(f"{field} must be numeric.") + number = float(value) + if not math.isfinite(number) or number < 0: + raise GateError(f"{field} must be a finite non-negative number.") + return number + + +def positive_integer(value: Any, field: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise GateError(f"{field} must be a positive integer.") + return value + + +def load_policy(path: Path) -> Policy: + payload = read_json(path, "benchmark policy") + if payload.get("schema_version") != 1: + raise GateError("benchmark policy schema_version must be 1.") + + suite = payload.get("benchmark_suite") + if not isinstance(suite, str) or not suite: + raise GateError("benchmark_suite must be a non-empty string.") + + baseline_revision = payload.get("baseline_revision") + legacy_revision = payload.get("legacy_baseline_revision") + if not isinstance(baseline_revision, int) or baseline_revision < 0: + raise GateError("baseline_revision must be a non-negative integer.") + if not isinstance(legacy_revision, int) or legacy_revision < 0: + raise GateError("legacy_baseline_revision must be a non-negative integer.") + + definitions = payload.get("benchmark_definition") + if not isinstance(definitions, dict): + raise GateError("benchmark_definition must be an object.") + + roots_value = definitions.get("roots") + files_value = definitions.get("files") + if not isinstance(roots_value, list) or not roots_value: + raise GateError("benchmark_definition.roots must be a non-empty array.") + if not isinstance(files_value, list): + raise GateError("benchmark_definition.files must be an array.") + + roots = tuple(safe_repository_path(value, "benchmark_definition.roots") for value in roots_value) + files = tuple(safe_repository_path(value, "benchmark_definition.files") for value in files_value) + + groups_value = payload.get("groups") + if not isinstance(groups_value, list) or not groups_value: + raise GateError("groups must be a non-empty array.") + + groups: list[ThresholdGroup] = [] + group_names: set[str] = set() + for index, group_value in enumerate(groups_value): + field = f"groups[{index}]" + if not isinstance(group_value, dict): + raise GateError(f"{field} must be an object.") + + name = group_value.get("name") + patterns_value = group_value.get("patterns") + if not isinstance(name, str) or not name: + raise GateError(f"{field}.name must be a non-empty string.") + if name in group_names: + raise GateError(f"Duplicate threshold group '{name}'.") + if not isinstance(patterns_value, list) or not patterns_value: + raise GateError(f"{field}.patterns must be a non-empty array.") + if not all(isinstance(pattern, str) and pattern for pattern in patterns_value): + raise GateError(f"{field}.patterns must contain non-empty strings.") + + warning = numeric_percent(group_value.get("warning_percent"), f"{field}.warning_percent") + failure = numeric_percent(group_value.get("failure_percent"), f"{field}.failure_percent") + observed = numeric_percent( + group_value.get("observed_max_slowdown_percent"), + f"{field}.observed_max_slowdown_percent", + ) + if warning <= observed: + raise GateError( + f"{field}.warning_percent must exceed its observed normal slowdown ({observed:.1f}%)." + ) + if failure <= warning: + raise GateError(f"{field}.failure_percent must exceed warning_percent.") + if failure >= 200: + raise GateError(f"{field}.failure_percent must remain below the retired 200% ceiling.") + + groups.append( + ThresholdGroup( + name=name, + patterns=tuple(patterns_value), + warning_percent=warning, + failure_percent=failure, + observed_max_slowdown_percent=observed, + ) + ) + group_names.add(name) + + return Policy( + benchmark_suite=suite, + baseline_revision=baseline_revision, + legacy_baseline_revision=legacy_revision, + minimum_compatible_samples=positive_integer( + payload.get("minimum_compatible_samples"), + "minimum_compatible_samples", + ), + history_window=positive_integer(payload.get("history_window"), "history_window"), + definition_roots=roots, + definition_files=files, + groups=tuple(groups), + ) + + +def parse_positive_number(value: Any, field: str) -> float: + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise GateError(f"{field} must be numeric.") + number = float(value) + if not math.isfinite(number) or number <= 0: + raise GateError(f"{field} must be a finite positive number.") + return number + + +def load_current_report(path: Path) -> dict[str, BenchmarkValue]: + payload = read_json(path, "current benchmark report") + benchmarks_value = payload.get("Benchmarks") + if not isinstance(benchmarks_value, list) or not benchmarks_value: + raise GateError("Current benchmark report must contain a non-empty Benchmarks array.") + + benchmarks: dict[str, BenchmarkValue] = {} + for index, item in enumerate(benchmarks_value): + if not isinstance(item, dict): + raise GateError(f"Benchmarks[{index}] must be an object.") + name = item.get("FullName") + statistics_value = item.get("Statistics") + if not isinstance(name, str) or not name: + raise GateError(f"Benchmarks[{index}].FullName must be a non-empty string.") + if not isinstance(statistics_value, dict): + raise GateError(f"Benchmarks[{index}].Statistics must be an object.") + if name in benchmarks: + raise GateError(f"Current benchmark report contains duplicate benchmark '{name}'.") + + benchmarks[name] = BenchmarkValue( + name=name, + value=parse_positive_number( + statistics_value.get("Mean"), + f"Benchmarks[{index}].Statistics.Mean", + ), + unit="ns", + ) + return benchmarks + + +def load_history(path: Path, suite: str) -> list[HistoryEntry]: + try: + contents = path.read_text(encoding="utf-8").strip() + except OSError as error: + raise GateError(f"Unable to read benchmark history '{path}': {error}") from error + + if not contents.startswith(BENCHMARK_DATA_PREFIX): + raise GateError("Benchmark history must start with the expected BENCHMARK_DATA assignment.") + json_contents = contents[len(BENCHMARK_DATA_PREFIX) :].strip() + if json_contents.endswith(";"): + json_contents = json_contents[:-1].rstrip() + + try: + payload = json.loads(json_contents) + except json.JSONDecodeError as error: + raise GateError(f"Benchmark history is not valid JSON data: {error}") from error + if not isinstance(payload, dict): + raise GateError("Benchmark history payload must be an object.") + + entries_value = payload.get("entries") + if not isinstance(entries_value, dict): + raise GateError("Benchmark history must contain an entries object.") + suite_entries = entries_value.get(suite) + if not isinstance(suite_entries, list): + raise GateError(f"Benchmark history does not contain suite '{suite}'.") + + entries: list[HistoryEntry] = [] + for index, entry_value in enumerate(suite_entries): + if not isinstance(entry_value, dict): + raise GateError(f"History entry {index} must be an object.") + if entry_value.get("tool") != "benchmarkdotnet": + raise GateError(f"History entry {index} is not a BenchmarkDotNet result.") + commit_value = entry_value.get("commit") + commit = commit_value.get("id") if isinstance(commit_value, dict) else None + if not isinstance(commit, str) or not COMMIT_PATTERN.fullmatch(commit): + raise GateError(f"History entry {index} has an invalid full commit SHA.") + date = parse_positive_number(entry_value.get("date"), f"History entry {index}.date") + benches_value = entry_value.get("benches") + if not isinstance(benches_value, list) or not benches_value: + raise GateError(f"History entry {index} must contain benchmark values.") + + benchmarks: list[BenchmarkValue] = [] + names: set[str] = set() + for bench_index, benchmark_value in enumerate(benches_value): + if not isinstance(benchmark_value, dict): + raise GateError(f"History entry {index} benchmark {bench_index} must be an object.") + name = benchmark_value.get("name") + unit = benchmark_value.get("unit") + if not isinstance(name, str) or not name: + raise GateError(f"History entry {index} benchmark {bench_index} has no name.") + if not isinstance(unit, str) or not unit: + raise GateError(f"History entry {index} benchmark {bench_index} has no unit.") + if name in names: + raise GateError(f"History entry {index} contains duplicate benchmark '{name}'.") + benchmarks.append( + BenchmarkValue( + name=name, + value=parse_positive_number( + benchmark_value.get("value"), + f"History entry {index} benchmark {bench_index}.value", + ), + unit=unit, + ) + ) + names.add(name) + + entries.append(HistoryEntry(commit=commit, date=date, benchmarks=tuple(benchmarks))) + + return sorted(entries, key=lambda entry: entry.date) + + +def run_git(repository_root: Path, arguments: list[str], *, allow_failure: bool = False) -> bytes | None: + result = subprocess.run( + ["git", *arguments], + cwd=repository_root, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if result.returncode == 0: + return result.stdout + if allow_failure: + return None + stderr = result.stderr.decode("utf-8", errors="replace").strip() + raise GateError(f"git {' '.join(arguments)} failed: {stderr or 'unknown error'}") + + +def fingerprint_contents(contents: dict[str, bytes]) -> str: + digest = hashlib.sha256() + for path in sorted(contents): + digest.update(path.encode("utf-8")) + digest.update(b"\0") + digest.update(contents[path]) + digest.update(b"\0") + return digest.hexdigest() + + +def current_definition_fingerprint(repository_root: Path, policy: Policy) -> str: + selectors = [*policy.definition_roots, *policy.definition_files] + output = run_git( + repository_root, + ["ls-files", "-z", "--cached", "--others", "--exclude-standard", "--", *selectors], + ) + assert output is not None + paths = sorted( + path.decode("utf-8") + for path in output.split(b"\0") + if path + ) + for required_file in policy.definition_files: + if required_file not in paths: + raise GateError(f"Benchmark definition file '{required_file}' is missing.") + for root in policy.definition_roots: + prefix = f"{root.rstrip('/')}/" + if not any(path == root or path.startswith(prefix) for path in paths): + raise GateError(f"Benchmark definition root '{root}' contains no files.") + + contents: dict[str, bytes] = {} + for path in paths: + resolved = (repository_root / path).resolve() + try: + resolved.relative_to(repository_root) + except ValueError as error: + raise GateError(f"Benchmark definition path '{path}' escapes the repository.") from error + try: + contents[path] = resolved.read_bytes() + except OSError as error: + raise GateError(f"Unable to read benchmark definition '{path}': {error}") from error + return fingerprint_contents(contents) + + +def historical_definition_fingerprint( + repository_root: Path, + commit: str, + policy: Policy, +) -> str | None: + selectors = [*policy.definition_roots, *policy.definition_files] + output = run_git( + repository_root, + ["ls-tree", "-r", "--name-only", "-z", commit, "--", *selectors], + allow_failure=True, + ) + if output is None: + return None + + paths = sorted( + path.decode("utf-8") + for path in output.split(b"\0") + if path + ) + if any(required_file not in paths for required_file in policy.definition_files): + return None + for root in policy.definition_roots: + prefix = f"{root.rstrip('/')}/" + if not any(path == root or path.startswith(prefix) for path in paths): + return None + + contents: dict[str, bytes] = {} + for path in paths: + content = run_git( + repository_root, + ["show", f"{commit}:{path}"], + allow_failure=True, + ) + if content is None: + return None + contents[path] = content + return fingerprint_contents(contents) + + +def historical_baseline_revision( + repository_root: Path, + commit: str, + policy_relative_path: str, + legacy_revision: int, +) -> int | None: + contents = run_git( + repository_root, + ["show", f"{commit}:{policy_relative_path}"], + allow_failure=True, + ) + if contents is None: + return legacy_revision + try: + payload = json.loads(contents.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + revision = payload.get("baseline_revision") if isinstance(payload, dict) else None + return revision if isinstance(revision, int) and revision >= 0 else None + + +def compatible_history( + repository_root: Path, + policy_path: Path, + policy: Policy, + entries: list[HistoryEntry], + baseline_ancestor: str, +) -> tuple[list[HistoryEntry], dict[str, int], str]: + if not COMMIT_PATTERN.fullmatch(baseline_ancestor): + raise GateError("baseline ancestor must be a full 40-character lowercase commit SHA.") + baseline_exists = run_git( + repository_root, + ["cat-file", "-e", f"{baseline_ancestor}^{{commit}}"], + allow_failure=True, + ) + if baseline_exists is None: + raise GateError(f"baseline ancestor commit '{baseline_ancestor}' is unavailable.") + + try: + policy_relative_path = policy_path.resolve().relative_to(repository_root).as_posix() + except ValueError as error: + raise GateError("Benchmark policy must be inside the repository root.") from error + + current_fingerprint = current_definition_fingerprint(repository_root, policy) + compatible: list[HistoryEntry] = [] + exclusions = { + "missing_commit": 0, + "not_base_ancestor": 0, + "definition_mismatch": 0, + "revision_mismatch": 0, + } + fingerprint_cache: dict[str, str | None] = {} + revision_cache: dict[str, int | None] = {} + + for entry in entries: + commit_exists = run_git( + repository_root, + ["cat-file", "-e", f"{entry.commit}^{{commit}}"], + allow_failure=True, + ) + if commit_exists is None: + exclusions["missing_commit"] += 1 + continue + + is_base_ancestor = run_git( + repository_root, + ["merge-base", "--is-ancestor", entry.commit, baseline_ancestor], + allow_failure=True, + ) + if is_base_ancestor is None: + exclusions["not_base_ancestor"] += 1 + continue + + if entry.commit not in fingerprint_cache: + fingerprint_cache[entry.commit] = historical_definition_fingerprint( + repository_root, + entry.commit, + policy, + ) + if fingerprint_cache[entry.commit] != current_fingerprint: + exclusions["definition_mismatch"] += 1 + continue + + if entry.commit not in revision_cache: + revision_cache[entry.commit] = historical_baseline_revision( + repository_root, + entry.commit, + policy_relative_path, + policy.legacy_baseline_revision, + ) + if revision_cache[entry.commit] != policy.baseline_revision: + exclusions["revision_mismatch"] += 1 + continue + + compatible.append(entry) + + return compatible[-policy.history_window :], exclusions, current_fingerprint + + +def resolve_group(name: str, groups: tuple[ThresholdGroup, ...]) -> ThresholdGroup: + matches = [ + group + for group in groups + if any(fnmatch.fnmatchcase(name, pattern) for pattern in group.patterns) + ] + if len(matches) != 1: + matched_names = ", ".join(group.name for group in matches) or "none" + raise GateError( + f"Benchmark '{name}' must match exactly one threshold group; matched: {matched_names}." + ) + return matches[0] + + +def meets_threshold(change_percent: float, threshold_percent: float) -> bool: + return change_percent > threshold_percent or math.isclose( + change_percent, + threshold_percent, + rel_tol=1e-12, + abs_tol=1e-9, + ) + + +def evaluate( + current: dict[str, BenchmarkValue], + entries: list[HistoryEntry], + policy: Policy, +) -> list[Outcome]: + outcomes: list[Outcome] = [] + matched_groups: set[str] = set() + for name in sorted(current): + measurement = current[name] + group = resolve_group(name, policy.groups) + matched_groups.add(group.name) + samples = [ + benchmark.value + for entry in entries + for benchmark in entry.benchmarks + if benchmark.name == name and benchmark.unit == measurement.unit + ] + + if len(samples) < policy.minimum_compatible_samples: + outcomes.append( + Outcome( + benchmark=name, + group=group.name, + sample_count=len(samples), + baseline=None, + current=measurement.value, + unit=measurement.unit, + change_percent=None, + warning_percent=group.warning_percent, + failure_percent=group.failure_percent, + status="WARMUP", + ) + ) + continue + + baseline = statistics.median(samples) + change_percent = ((measurement.value / baseline) - 1.0) * 100.0 + if meets_threshold(change_percent, group.failure_percent): + status = "FAIL" + elif meets_threshold(change_percent, group.warning_percent): + status = "WARNING" + else: + status = "PASS" + + outcomes.append( + Outcome( + benchmark=name, + group=group.name, + sample_count=len(samples), + baseline=baseline, + current=measurement.value, + unit=measurement.unit, + change_percent=change_percent, + warning_percent=group.warning_percent, + failure_percent=group.failure_percent, + status=status, + ) + ) + + unmatched_groups = [group.name for group in policy.groups if group.name not in matched_groups] + if unmatched_groups: + raise GateError( + "Threshold groups matched no current benchmarks: " + + ", ".join(unmatched_groups) + + "." + ) + return outcomes + + +def markdown_cell(value: str) -> str: + return value.replace("\\", "\\\\").replace("|", "\\|").replace("\n", " ") + + +def format_measurement(value: float | None, unit: str) -> str: + if value is None: + return "—" + return f"{value:,.2f} {unit}" + + +def build_summary( + outcomes: list[Outcome], + compatible_count: int, + exclusions: dict[str, int], + fingerprint: str, + policy: Policy, + allow_failure: bool, + baseline_ancestor: str, +) -> str: + failures = [outcome for outcome in outcomes if outcome.status == "FAIL"] + warnings = [outcome for outcome in outcomes if outcome.status == "WARNING"] + warmups = [outcome for outcome in outcomes if outcome.status == "WARMUP"] + + if failures: + overall = "FAIL" + marker = "❌" + elif warnings: + overall = "WARNING" + marker = "⚠️" + elif warmups: + overall = "WARMUP" + marker = "⚠️" + else: + overall = "PASS" + marker = "✅" + + lines = [ + "## Performance regression gate", + "", + f"{marker} **{overall}** — {compatible_count} compatible hosted-runner sample(s); " + f"median baseline; revision `{policy.baseline_revision}`.", + "", + f"- Intended base commit: `{baseline_ancestor}`", + f"- Definition fingerprint: `{fingerprint[:16]}`", + f"- History window: newest {policy.history_window} compatible samples", + f"- Excluded history: definition mismatch {exclusions['definition_mismatch']}, " + f"revision mismatch {exclusions['revision_mismatch']}, " + f"not an intended-base ancestor {exclusions['not_base_ancestor']}, " + f"missing commit {exclusions['missing_commit']}", + ] + if allow_failure: + lines.append( + "- Intentional baseline publication mode is enabled: failures remain visible but do not block this run." + ) + if warmups: + lines.append( + f"- {len(warmups)} benchmark(s) have fewer than " + f"{policy.minimum_compatible_samples} compatible samples and are reported as WARMUP." + ) + + lines.extend( + [ + "", + "| Benchmark | Group | Samples | Baseline median | Current mean | Change | Warning | Failure | Result |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |", + ] + ) + for outcome in outcomes: + change = "—" if outcome.change_percent is None else f"{outcome.change_percent:+.1f}%" + lines.append( + "| " + + " | ".join( + [ + markdown_cell(outcome.benchmark), + markdown_cell(outcome.group), + str(outcome.sample_count), + format_measurement(outcome.baseline, outcome.unit), + format_measurement(outcome.current, outcome.unit), + change, + f"{outcome.warning_percent:.0f}%", + f"{outcome.failure_percent:.0f}%", + outcome.status, + ] + ) + + " |" + ) + + lines.extend( + [ + "", + f"Warnings: {len(warnings)}. Failures: {len(failures)}. Warmups: {len(warmups)}.", + ] + ) + return "\n".join(lines) + "\n" + + +def build_error_summary(error: GateError) -> str: + return "\n".join( + [ + "## Performance regression gate", + "", + f"❌ **CONFIGURATION ERROR:** {error}", + "", + ] + ) + + +def append_summary(path_value: str | None, summary: str) -> None: + if not path_value: + return + path = Path(path_value) + try: + with path.open("a", encoding="utf-8") as stream: + stream.write(summary) + except OSError as error: + raise GateError(f"Unable to append GitHub job summary '{path}': {error}") from error + + +def emit_annotations(outcomes: list[Outcome]) -> None: + for outcome in outcomes: + if outcome.status not in {"WARNING", "FAIL"}: + continue + command = "error" if outcome.status == "FAIL" else "warning" + change = outcome.change_percent if outcome.change_percent is not None else 0.0 + print( + f"::{command} title=Benchmark {outcome.status}::{outcome.benchmark} " + f"slowed by {change:.1f}% " + f"({outcome.warning_percent:.0f}% warning / {outcome.failure_percent:.0f}% failure)." + ) + + +def main() -> int: + args = parse_args() + summary_path = args.summary or os.environ.get("GITHUB_STEP_SUMMARY") + allow_failure = args.allow_failure == "true" + + try: + repository_root = Path(args.repository_root).resolve() + policy_path = Path(args.policy).resolve() + policy = load_policy(policy_path) + current = load_current_report(Path(args.current_report)) + history = load_history(Path(args.history_data), policy.benchmark_suite) + compatible, exclusions, fingerprint = compatible_history( + repository_root, + policy_path, + policy, + history, + args.baseline_ancestor, + ) + outcomes = evaluate(current, compatible, policy) + summary = build_summary( + outcomes, + len(compatible), + exclusions, + fingerprint, + policy, + allow_failure, + args.baseline_ancestor, + ) + print(summary, end="") + emit_annotations(outcomes) + append_summary(summary_path, summary) + except GateError as error: + summary = build_error_summary(error) + print(summary, file=sys.stderr, end="") + try: + append_summary(summary_path, summary) + except GateError as summary_error: + print(summary_error, file=sys.stderr) + return 2 + + has_failure = any(outcome.status == "FAIL" for outcome in outcomes) + return 0 if allow_failure or not has_failure else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_check_benchmark_regressions.py b/scripts/tests/test_check_benchmark_regressions.py new file mode 100644 index 00000000..66d109af --- /dev/null +++ b/scripts/tests/test_check_benchmark_regressions.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +""" +Tests the evidence-based benchmark regression gate. +実測ベースのベンチマーク回帰ゲートをテストします。 +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +SCRIPT_PATH = Path(__file__).resolve().parent.parent / "check_benchmark_regressions.py" +BENCHMARK_NAME = "Example.Benchmarks.Sample" + + +class BenchmarkRegressionGateTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.root = Path(self.temporary_directory.name) + (self.root / "bench").mkdir() + (self.root / "bench" / "Benchmark.cs").write_text( + "public static class Benchmark {}\n", + encoding="utf-8", + ) + self.policy_path = self.root / "benchmark-regression-policy.json" + self.write_policy() + self.run_git("init") + self.run_git("config", "user.name", "Benchmark Test") + self.run_git("config", "user.email", "benchmark-test@example.invalid") + self.run_git("config", "commit.gpgSign", "false") + self.run_git("add", "bench/Benchmark.cs") + self.run_git("commit", "-m", "Add benchmark definition") + self.commit = self.run_git("rev-parse", "HEAD").stdout.strip() + + def tearDown(self) -> None: + self.temporary_directory.cleanup() + + def run_git(self, *arguments: str) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + ["git", *arguments], + cwd=self.root, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(0, result.returncode, result.stderr) + return result + + def write_policy(self, *, revision: int = 0, include_pattern: bool = True) -> None: + pattern = "*" if include_pattern else "Different.*" + policy = { + "schema_version": 1, + "benchmark_suite": "Fixture Suite", + "baseline_revision": revision, + "legacy_baseline_revision": 0, + "minimum_compatible_samples": 3, + "history_window": 5, + "benchmark_definition": { + "roots": ["bench"], + "files": [], + }, + "groups": [ + { + "name": "fixture", + "patterns": [pattern], + "observed_max_slowdown_percent": 10, + "warning_percent": 20, + "failure_percent": 40, + } + ], + } + self.policy_path.write_text(json.dumps(policy), encoding="utf-8") + + def write_current_report(self, value: float) -> Path: + path = self.root / "current.json" + path.write_text( + json.dumps( + { + "Benchmarks": [ + { + "FullName": BENCHMARK_NAME, + "Statistics": {"Mean": value}, + } + ] + } + ), + encoding="utf-8", + ) + return path + + def write_history(self, values: list[float], commit: str | None = None) -> Path: + path = self.root / "data.js" + entries = [] + for index, value in enumerate(values): + entries.append( + { + "commit": {"id": commit or self.commit}, + "date": 1_700_000_000_000 + index, + "tool": "benchmarkdotnet", + "benches": [ + { + "name": BENCHMARK_NAME, + "value": value, + "unit": "ns", + } + ], + } + ) + payload = { + "entries": { + "Fixture Suite": entries, + } + } + path.write_text( + f"window.BENCHMARK_DATA = {json.dumps(payload)}", + encoding="utf-8", + ) + return path + + def run_gate( + self, + current_value: float, + history_values: list[float], + *, + allow_failure: bool = False, + history_commit: str | None = None, + baseline_ancestor: str | None = None, + ) -> subprocess.CompletedProcess[str]: + current_path = self.write_current_report(current_value) + history_path = self.write_history(history_values, history_commit) + summary_path = self.root / "summary.md" + return subprocess.run( + [ + sys.executable, + str(SCRIPT_PATH), + "--current-report", + str(current_path), + "--history-data", + str(history_path), + "--policy", + str(self.policy_path), + "--baseline-ancestor", + baseline_ancestor or self.commit, + "--repository-root", + str(self.root), + "--summary", + str(summary_path), + "--allow-failure", + str(allow_failure).lower(), + ], + cwd=self.root, + capture_output=True, + text=True, + check=False, + ) + + def test_median_threshold_boundaries_produce_pass_warning_and_failure(self) -> None: + history = [98, 100, 102] + + passing = self.run_gate(119.9, history) + warning = self.run_gate(120, history) + failure = self.run_gate(140, history) + + self.assertEqual(0, passing.returncode, passing.stderr) + self.assertIn("| PASS |", passing.stdout) + self.assertEqual(0, warning.returncode, warning.stderr) + self.assertIn("| WARNING |", warning.stdout) + self.assertIn("::warning", warning.stdout) + self.assertEqual(1, failure.returncode, failure.stderr) + self.assertIn("| FAIL |", failure.stdout) + self.assertIn("::error", failure.stdout) + + def test_failure_can_be_reported_without_blocking_intentional_baseline_publication(self) -> None: + result = self.run_gate(150, [98, 100, 102], allow_failure=True) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("Intentional baseline publication mode is enabled", result.stdout) + self.assertIn("| FAIL |", result.stdout) + + def test_definition_mismatch_is_visible_and_enters_warmup(self) -> None: + (self.root / "bench" / "Benchmark.cs").write_text( + "public static class ChangedBenchmark {}\n", + encoding="utf-8", + ) + + result = self.run_gate(500, [98, 100, 102]) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("definition mismatch 3", result.stdout) + self.assertIn("| WARMUP |", result.stdout) + + def test_baseline_revision_change_excludes_old_history(self) -> None: + self.write_policy(revision=1) + + result = self.run_gate(500, [98, 100, 102]) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("revision mismatch 3", result.stdout) + self.assertIn("| WARMUP |", result.stdout) + + def test_history_newer_than_intended_base_is_excluded(self) -> None: + (self.root / "after-base.txt").write_text("newer\n", encoding="utf-8") + self.run_git("add", "after-base.txt") + self.run_git("commit", "-m", "Create a commit after the intended base") + newer_commit = self.run_git("rev-parse", "HEAD").stdout.strip() + + result = self.run_gate( + 500, + [98, 100, 102], + history_commit=newer_commit, + baseline_ancestor=self.commit, + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("not an intended-base ancestor 3", result.stdout) + self.assertIn("| WARMUP |", result.stdout) + + def test_unassigned_benchmark_fails_closed(self) -> None: + self.write_policy(include_pattern=False) + + result = self.run_gate(100, [98, 100, 102]) + + self.assertEqual(2, result.returncode) + self.assertIn("must match exactly one threshold group", result.stderr) + + def test_policy_rejects_retired_200_percent_failure_ceiling(self) -> None: + policy = json.loads(self.policy_path.read_text(encoding="utf-8")) + policy["groups"][0]["failure_percent"] = 200 + self.policy_path.write_text(json.dumps(policy), encoding="utf-8") + + result = self.run_gate(100, [98, 100, 102]) + + self.assertEqual(2, result.returncode) + self.assertIn("must remain below the retired 200% ceiling", result.stderr) + + +if __name__ == "__main__": + unittest.main()