From 7c2ebbc6365d774470536e1e74334d982d8baf23 Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 20:09:18 +0800 Subject: [PATCH 1/4] Align benchmark skill submission routing --- .agents/skills/run-api-benchmark/SKILL.md | 13 ++++++------- .agents/skills/run-benchmark/SKILL.md | 4 ++++ .agents/skills/run-cli-benchmark/SKILL.md | 16 ++++++++-------- benchmark/tests/test_docs.py | 13 ++++++++++--- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/.agents/skills/run-api-benchmark/SKILL.md b/.agents/skills/run-api-benchmark/SKILL.md index d0113c9..d2dfb4f 100644 --- a/.agents/skills/run-api-benchmark/SKILL.md +++ b/.agents/skills/run-api-benchmark/SKILL.md @@ -38,11 +38,10 @@ Do not dump every configuration question into one message. > What should happen after the run? > > 1. Keep and validate the result locally without uploading. - > 2. Upload an intake test that is scored privately but excluded from the leaderboard. - > 3. Upload an official submission. + > 2. Upload an official submission. Default to local-only only when the caller explicitly delegates the choice. The - `$submit-benchmark-result` skill owns intake authentication and upload. + `$submit-benchmark-result` skill owns submission validation, authentication, and upload. 4. Resolve `PR_REF` (default `v0.6.0`), `SUBMIT_LIMIT` (default 100), and `STAMP` (default: the Makefile timestamp). Show the derived authoritative path @@ -93,7 +92,7 @@ credits and ask for explicit confirmation to start it. Then run `make run` or th Podman command using the detector's `RUN_FLAGS`. Pass `STAMP=` when a fixed stamp was selected. -Confirm the authoritative `submission.json` exists. Always validate it: +Confirm the authoritative `submission.json` exists. For option 1, validate it locally: ```bash python -m benchmark.submit --predictions --dry-run @@ -102,9 +101,9 @@ python -m benchmark.submit --predictions --dry-run Report `bugs_found`, `total_tokens_k`, submit attempts, any `run_error`, and absolute output and log paths. A `run_error` means partial salvage, not a clean zero-bug completion. -For option 2 or 3, invoke `$submit-benchmark-result` with the authoritative path and the -already-selected test/official mode. That skill owns authentication, final confirmation, -upload, and submission-ID reporting. Never upload merely because the run completed. +For option 2, invoke `$submit-benchmark-result` with the authoritative path. Do not validate +it first: that skill owns validation, authentication, final confirmation, upload, scoring, +and PR reporting. Never upload merely because the run completed. An exit code 137 means the engine needs more memory. Preserve partial outputs and read actual command errors before recommending changes. diff --git a/.agents/skills/run-benchmark/SKILL.md b/.agents/skills/run-benchmark/SKILL.md index 3d8f13a..4dab277 100644 --- a/.agents/skills/run-benchmark/SKILL.md +++ b/.agents/skills/run-benchmark/SKILL.md @@ -5,6 +5,10 @@ description: Route a request to run, reproduce, smoke-test, or generate a submis # Route a benchmark run +If the caller already has a `submission.json` and wants to validate or submit it, invoke +`$submit-benchmark-result` and stop. Do not ask how to run a model when the result already +exists. + Choose exactly one execution route before asking about models, credentials, paths, or submission settings. diff --git a/.agents/skills/run-cli-benchmark/SKILL.md b/.agents/skills/run-cli-benchmark/SKILL.md index ecd00b6..d7c89a9 100644 --- a/.agents/skills/run-cli-benchmark/SKILL.md +++ b/.agents/skills/run-cli-benchmark/SKILL.md @@ -52,10 +52,10 @@ numbered stage at a time when its answer controls the next branch. > What should happen after the run? > > 1. Keep and validate the result locally without uploading. - > 2. Upload an intake test that is scored privately but excluded from the leaderboard. - > 3. Upload an official submission. + > 2. Upload an official submission. - The `$submit-benchmark-result` skill owns intake authentication and upload. + The `$submit-benchmark-result` skill owns submission validation, authentication, and + upload. 5. Resolve `PR_REF` (default `v0.6.0`), `SUBMIT_LIMIT` (default 100), and three separate, explicit paths: clone destination, authoritative submission JSON, and log directory. @@ -87,7 +87,7 @@ Before the first real model call, show: - target `PR_REF` and resolved commit; - submit limit; - clone destination, authoritative output path, and log path; -- local/test versus official-upload goal. +- local versus official-upload goal. State that the run can consume substantial time or credits, then get explicit confirmation. @@ -114,7 +114,7 @@ attempt reaches the service, require `run_error` and inspect ## Validate and hand off -Always validate the authoritative file: +For option 1, validate the authoritative file locally: ```bash python -m benchmark.submit --predictions --dry-run @@ -123,6 +123,6 @@ python -m benchmark.submit --predictions --dry-run Report `bugs_found`, `total_tokens_k`, submit attempts, any `run_error`, CLI warnings, and absolute output/log paths. Preserve partial results and logs on failure. -For option 2 or 3, invoke `$submit-benchmark-result` with the authoritative path and the -already-selected test/official mode. That skill owns authentication, final confirmation, -upload, and submission-ID reporting. Never upload merely because the run completed. +For option 2, invoke `$submit-benchmark-result` with the authoritative path. Do not validate +it first: that skill owns validation, authentication, final confirmation, upload, scoring, +and PR reporting. Never upload merely because the run completed. diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index de738f4..6fab5eb 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -113,15 +113,22 @@ def test_cli_skill_is_host_only(self): assert "or start docker/podman" in t @pytest.mark.parametrize("skill", [API_SKILL, CLI_SKILL]) - def test_each_skill_exposes_three_submission_goals(self, skill): + def test_each_skill_exposes_only_local_and_official_goals(self, skill): t = _text(skill) assert "keep and validate" in t - assert "intake test" in t assert "official submission" in t + assert "intake test" not in t @pytest.mark.parametrize("skill", [API_SKILL, CLI_SKILL]) def test_run_skills_delegate_upload(self, skill): - assert "$submit-benchmark-result" in _text(skill) + t = _text(skill) + assert "$submit-benchmark-result" in t + assert "that skill owns validation" in t + + def test_router_sends_existing_results_to_submit_skill(self): + t = _text(REPO_ROOT / ".agents/skills/run-benchmark/SKILL.md") + assert "already has a `submission.json`" in t + assert "$submit-benchmark-result" in t class TestSubmitSkill: From 6de2ff52f7ed763b468e7fa1b0396df83ca1d76b Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 20:15:49 +0800 Subject: [PATCH 2/4] Read benchmark round from Makefile --- .agents/skills/run-api-benchmark/SKILL.md | 13 +++++++------ .agents/skills/run-cli-benchmark/SKILL.md | 11 ++++++----- Makefile | 6 +++++- benchmark/tests/test_docs.py | 12 ++++++++++++ 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/.agents/skills/run-api-benchmark/SKILL.md b/.agents/skills/run-api-benchmark/SKILL.md index d2dfb4f..974867f 100644 --- a/.agents/skills/run-api-benchmark/SKILL.md +++ b/.agents/skills/run-api-benchmark/SKILL.md @@ -43,8 +43,9 @@ Do not dump every configuration question into one message. Default to local-only only when the caller explicitly delegates the choice. The `$submit-benchmark-result` skill owns submission validation, authentication, and upload. -4. Resolve `PR_REF` (default `v0.6.0`), `SUBMIT_LIMIT` (default 100), and `STAMP` (default: - the Makefile timestamp). Show the derived authoritative path +4. Read the current benchmark round with `make -s print-pr-ref`; use that `PR_REF` unless the + caller explicitly requested an override. Resolve `SUBMIT_LIMIT` (default 100) and `STAMP` + (default: the Makefile timestamp). Show the derived authoritative path `out//submission.json`; the trajectory is written alongside it. Do not ask for arbitrary host output or log paths when using `make run`. @@ -73,12 +74,12 @@ Do not expose secret values in command output or the final response. Do not add 2. Prepare one image at the selected ref. Prefer the published image: ```bash - make runner-pull PR_REF=v0.6.0 + make runner-pull ``` - Fall back to `make runner-build PR_REF=v0.6.0` only when the published image is - unavailable or the caller explicitly wants a local build. For Podman, use the equivalent - command from `references/engines.md`. + Fall back to `make runner-build` only when the published image is unavailable or the + caller explicitly wants a local build. For Podman, use the equivalent command from + `references/engines.md`. 3. Before the preflight's real API call, show the resolved model, backend `mini-swe`, API endpoint with secrets redacted, `PR_REF`, submit limit, `STAMP`, derived submission path, and upload goal. Ask for explicit confirmation. diff --git a/.agents/skills/run-cli-benchmark/SKILL.md b/.agents/skills/run-cli-benchmark/SKILL.md index d7c89a9..c22ff1c 100644 --- a/.agents/skills/run-cli-benchmark/SKILL.md +++ b/.agents/skills/run-cli-benchmark/SKILL.md @@ -57,9 +57,10 @@ numbered stage at a time when its answer controls the next branch. The `$submit-benchmark-result` skill owns submission validation, authentication, and upload. -5. Resolve `PR_REF` (default `v0.6.0`), `SUBMIT_LIMIT` (default 100), and three separate, - explicit paths: clone destination, authoritative submission JSON, and log directory. - Offer concrete defaults and ask whether to accept them. +5. Read the current benchmark round with `make -s print-pr-ref`; use that `PR_REF` unless the + caller explicitly requested an override. Resolve `SUBMIT_LIMIT` (default 100) and three + separate, explicit paths: clone destination, authoritative submission JSON, and log + directory. Offer concrete defaults and ask whether to accept them. ## Verify the selected harness @@ -100,10 +101,10 @@ Run through `make run-local` with the selected backend. If invoking example: ```bash +PR_REF="$(make -s print-pr-ref)" make run-local \ - PR_REF=v0.6.0 \ LOCAL_BACKEND=codex \ - LOCAL_REPO_DIR=../runs/problem-reductions-v0.6.0 \ + LOCAL_REPO_DIR="../runs/problem-reductions-${PR_REF}" \ LOCAL_OUTPUT=../runs/results/submission.json \ LOCAL_LOG_DIR=../runs/logs ``` diff --git a/Makefile b/Makefile index 05be524..d6ef030 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,11 @@ LOCAL_LOG_DIR ?= REPO_URL ?= https://github.com/CodingThrust/problem-reductions.git LOCAL_ARGS = $(if $(LOCAL_BACKEND),--backend "$(LOCAL_BACKEND)") -.PHONY: test test-unit verify-calibration verify-judgment audit install-deps help runner-build runner-pull preflight run run-local score-local board publish-local serve +.PHONY: test test-unit verify-calibration verify-judgment audit install-deps help print-pr-ref runner-build runner-pull preflight run run-local score-local board publish-local serve + +## Print the current benchmark round's problem-reductions ref for tools and skills. +print-pr-ref: + @echo "$(PR_REF)" ## Run the full test suite (unit + integration tests that need real repo). test: diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index 6fab5eb..8a78254 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -125,6 +125,18 @@ def test_run_skills_delegate_upload(self, skill): assert "$submit-benchmark-result" in t assert "that skill owns validation" in t + @pytest.mark.parametrize("skill", [API_SKILL, CLI_SKILL]) + def test_run_skills_read_the_round_from_make(self, skill): + t = _text(skill) + assert "make -s print-pr-ref" in t + assert "v0.6.0" not in t + + def test_make_exposes_the_current_round(self): + result = subprocess.run( + ["make", "-s", "print-pr-ref"], cwd=REPO_ROOT, + text=True, capture_output=True, check=True) + assert result.stdout.strip() + def test_router_sends_existing_results_to_submit_skill(self): t = _text(REPO_ROOT / ".agents/skills/run-benchmark/SKILL.md") assert "already has a `submission.json`" in t From 58af110539226a30edc89de54c440b7fad0f545d Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 20:28:49 +0800 Subject: [PATCH 3/4] Define benchmark version source of truth --- .agents/skills/run-api-benchmark/SKILL.md | 25 +++++++++++----- .agents/skills/run-cli-benchmark/SKILL.md | 22 ++++++++++---- .github/workflows/ci.yml | 9 +++--- .github/workflows/runner-image.yml | 9 +++--- .github/workflows/score-from-r2.yml | 9 +++--- Makefile | 14 ++++++--- README.md | 1 + VERSION | 1 + benchmark/tests/test_docs.py | 35 ++++++++++++++++++----- 9 files changed, 90 insertions(+), 35 deletions(-) create mode 100644 VERSION diff --git a/.agents/skills/run-api-benchmark/SKILL.md b/.agents/skills/run-api-benchmark/SKILL.md index 974867f..71db21f 100644 --- a/.agents/skills/run-api-benchmark/SKILL.md +++ b/.agents/skills/run-api-benchmark/SKILL.md @@ -43,11 +43,22 @@ Do not dump every configuration question into one message. Default to local-only only when the caller explicitly delegates the choice. The `$submit-benchmark-result` skill owns submission validation, authentication, and upload. -4. Read the current benchmark round with `make -s print-pr-ref`; use that `PR_REF` unless the - caller explicitly requested an override. Resolve `SUBMIT_LIMIT` (default 100) and `STAMP` - (default: the Makefile timestamp). Show the derived authoritative path - `out//submission.json`; the trajectory is written alongside it. Do not ask for - arbitrary host output or log paths when using `make run`. +4. Read this checkout's benchmark version with `make -s print-benchmark-version`. Read the + latest version from the official repository's + [`main/VERSION`](https://github.com/CodingThrust/problem-reductions-benchmark/blob/main/VERSION); + do not use the `problem-reductions` version or guess. Show this in the caller's language + and wait for confirmation: + + > Benchmark version: `` (latest version: `
`) + + If the versions differ, explain that the checkout is outdated and ask the caller to + update it before an official run. If the latest-version lookup fails, show `unknown` + rather than substituting the pinned `problem-reductions` version. + +5. Resolve the internal problem-reductions pin with `make -s print-pr-ref`, plus + `SUBMIT_LIMIT` (default 100) and `STAMP` (default: the Makefile timestamp). Show the + derived authoritative path `out//submission.json`; the trajectory is written + alongside it. Do not ask for arbitrary host output or log paths when using `make run`. ## Configure safely @@ -81,8 +92,8 @@ Do not expose secret values in command output or the final response. Do not add caller explicitly wants a local build. For Podman, use the equivalent command from `references/engines.md`. 3. Before the preflight's real API call, show the resolved model, backend `mini-swe`, API - endpoint with secrets redacted, `PR_REF`, submit limit, `STAMP`, derived submission path, - and upload goal. Ask for explicit confirmation. + endpoint with secrets redacted, confirmed benchmark version, submit limit, `STAMP`, + derived submission path, and upload goal. Ask for explicit confirmation. 4. Run `make preflight`. It checks `pred`, rule sources, and one tiny LiteLLM call. Stop on any failure; never proceed to a full run after a failed preflight. diff --git a/.agents/skills/run-cli-benchmark/SKILL.md b/.agents/skills/run-cli-benchmark/SKILL.md index c22ff1c..298bcec 100644 --- a/.agents/skills/run-cli-benchmark/SKILL.md +++ b/.agents/skills/run-cli-benchmark/SKILL.md @@ -57,10 +57,22 @@ numbered stage at a time when its answer controls the next branch. The `$submit-benchmark-result` skill owns submission validation, authentication, and upload. -5. Read the current benchmark round with `make -s print-pr-ref`; use that `PR_REF` unless the - caller explicitly requested an override. Resolve `SUBMIT_LIMIT` (default 100) and three - separate, explicit paths: clone destination, authoritative submission JSON, and log - directory. Offer concrete defaults and ask whether to accept them. +5. Read this checkout's benchmark version with `make -s print-benchmark-version`. Read the + latest version from the official repository's + [`main/VERSION`](https://github.com/CodingThrust/problem-reductions-benchmark/blob/main/VERSION); + do not use the `problem-reductions` version or guess. Show this in the caller's language + and wait for confirmation: + + > Benchmark version: `` (latest version: `
`) + + If the versions differ, explain that the checkout is outdated and ask the caller to + update it before an official run. If the latest-version lookup fails, show `unknown` + rather than substituting the pinned `problem-reductions` version. + +6. Resolve the internal problem-reductions pin with `make -s print-pr-ref`, plus + `SUBMIT_LIMIT` (default 100) and three separate, explicit paths: clone destination, + authoritative submission JSON, and log directory. Offer concrete defaults and ask + whether to accept them. ## Verify the selected harness @@ -85,7 +97,7 @@ Before the first real model call, show: - CLI human name and backend ID; - resolved CLI path/version and authentication status; - exact model; -- target `PR_REF` and resolved commit; +- confirmed benchmark version and resolved commit; - submit limit; - clone destination, authoritative output path, and log path; - local versus official-upload goal. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5873fea..a37dd58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,15 +5,16 @@ on: branches: [main] pull_request: -env: - PR_REF: v0.6.0 - jobs: test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + - name: Read benchmark version + id: version + run: echo "value=$(cat VERSION)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-python@v6 with: python-version: "3.12" @@ -55,7 +56,7 @@ jobs: context: . file: docker/Dockerfile target: runtime - build-args: PR_REF=${{ env.PR_REF }} + build-args: PR_REF=${{ steps.version.outputs.value }} load: true tags: prb-scoring:latest cache-from: type=gha diff --git a/.github/workflows/runner-image.yml b/.github/workflows/runner-image.yml index ebbb7e0..6cb3d9b 100644 --- a/.github/workflows/runner-image.yml +++ b/.github/workflows/runner-image.yml @@ -2,7 +2,7 @@ name: Runner image (GHCR) # Builds the bug-finding runner image (mini-swe/LiteLLM agent + pinned pred) and pushes it # to GHCR, so `make run` can start with `make runner-pull` instead of a ~30–60 min local -# Rust build. Tag = PR_REF from the Makefile (e.g. v0.6.0) + latest. +# Rust build. Tag = the benchmark version from VERSION (also the default PR_REF) + latest. # # One-time setup: after the first push, mark the package public # (Packages → problem-reductions-runner → Package settings) so anyone can pull it. @@ -14,7 +14,8 @@ on: - docker/Dockerfile - benchmark/requirements.txt - .github/workflows/runner-image.yml - - Makefile # PR_REF pin lives here + - Makefile + - VERSION workflow_dispatch: {} permissions: @@ -27,9 +28,9 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Read PR_REF from the Makefile + - name: Read benchmark version id: pins - run: echo "pr_ref=$(grep -E '^PR_REF[[:space:]]*\?=' Makefile | sed -E 's/.*=[[:space:]]*//')" >> "$GITHUB_OUTPUT" + run: echo "pr_ref=$(cat VERSION)" >> "$GITHUB_OUTPUT" # Docker repository names must be lowercase — the org login (CodingThrust) is not. - name: Lowercase GHCR image namespace diff --git a/.github/workflows/score-from-r2.yml b/.github/workflows/score-from-r2.yml index 7527d0e..9f56b64 100644 --- a/.github/workflows/score-from-r2.yml +++ b/.github/workflows/score-from-r2.yml @@ -23,9 +23,6 @@ permissions: contents: write # push the aggregate to a bot branch pull-requests: write # open the PR that updates the public leaderboard -env: - PR_REF: v0.6.0 - concurrency: group: score-from-r2 cancel-in-progress: false @@ -37,6 +34,10 @@ jobs: steps: - uses: actions/checkout@v7 + - name: Read benchmark version + id: version + run: echo "value=$(cat VERSION)" >> "$GITHUB_OUTPUT" + # buildx with the docker-container driver — required for the type=gha build cache # below (the default docker driver can't export cache). - name: Set up Docker Buildx @@ -48,7 +49,7 @@ jobs: context: . file: docker/Dockerfile target: runtime - build-args: PR_REF=${{ env.PR_REF }} + build-args: PR_REF=${{ steps.version.outputs.value }} load: true tags: prb-scoring:latest cache-from: type=gha diff --git a/Makefile b/Makefile index d6ef030..6ecb8f4 100644 --- a/Makefile +++ b/Makefile @@ -11,10 +11,12 @@ # # Model/auth configuration lives in submission.env (see submission.env.example). Local # repository, submission, and log locations are intentionally explicit Make variables. -# PR_REF = the problem-reductions version this benchmark round targets (tag or commit). -# It drives BOTH the build arg and the image tag, so bumping the round is one place: +# BENCHMARK_VERSION is the version of this benchmark contract. PR_REF is the +# problem-reductions version this round targets; by policy they are synchronized unless a +# local/debug run explicitly overrides PR_REF. They drive both the build arg and image tag: # make runner-build PR_REF=v0.7.0 → builds + tags problem-reductions-runner:v0.7.0 -PR_REF ?= v0.6.0 +BENCHMARK_VERSION := $(strip $(shell cat VERSION)) +PR_REF ?= $(BENCHMARK_VERSION) IMAGE ?= problem-reductions-runner:$(PR_REF) GHCR_IMAGE ?= ghcr.io/codingthrust/problem-reductions-runner JOBS ?= 1 @@ -29,7 +31,11 @@ LOCAL_LOG_DIR ?= REPO_URL ?= https://github.com/CodingThrust/problem-reductions.git LOCAL_ARGS = $(if $(LOCAL_BACKEND),--backend "$(LOCAL_BACKEND)") -.PHONY: test test-unit verify-calibration verify-judgment audit install-deps help print-pr-ref runner-build runner-pull preflight run run-local score-local board publish-local serve +.PHONY: test test-unit verify-calibration verify-judgment audit install-deps help print-benchmark-version print-pr-ref runner-build runner-pull preflight run run-local score-local board publish-local serve + +## Print this checkout's benchmark contract version. +print-benchmark-version: + @echo "$(BENCHMARK_VERSION)" ## Print the current benchmark round's problem-reductions ref for tools and skills. print-pr-ref: diff --git a/README.md b/README.md index c3253bb..4d3940a 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ The leaderboard is a static site (`site/`) published to **GitHub Pages**. Submit | Contract | Current value | Ownership | |---|---|---| +| Benchmark version | [`v0.6.0`](VERSION) | `VERSION` is the source of truth used by local commands and production workflows. | | Submission format | [`benchmark/submission.schema.json`](benchmark/submission.schema.json) | The runner writes the current structure and the scorer parses that structure directly. The payload has no schema-version field. | | `problem-reductions` target | [`v0.6.0` / `aa2d1a10cffa434871d12a4d6f411147fb7e08a8`](https://github.com/CodingThrust/problem-reductions/commit/aa2d1a10cffa434871d12a4d6f411147fb7e08a8) | Every official result is verified against this exact commit. | | `pred` | `0.6.0` | The runner/scorer image supplies and verifies the matching binary. | diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..60f6343 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +v0.6.0 diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index 8a78254..4d14f1f 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -25,6 +25,7 @@ SUBMISSIONS_README = REPO_ROOT / "submissions/README.md" SITE_INDEX = REPO_ROOT / "site/index.html" INTAKE_README = REPO_ROOT / "intake/cloudflare-worker/README.md" +VERSION_FILE = REPO_ROOT / "VERSION" def _text(path: Path) -> str: @@ -76,6 +77,8 @@ def test_readme_has_metrics_section(self): def test_readme_lists_current_round_contract(self): text = README.read_text(encoding="utf-8") + benchmark_version = VERSION_FILE.read_text(encoding="utf-8").strip() + assert f"[`{benchmark_version}`](VERSION)" in text assert PINNED_COMMIT in text assert f"`{PINNED_PRED_VERSION}`" in text assert "no schema-version field" in text.lower() @@ -126,16 +129,34 @@ def test_run_skills_delegate_upload(self, skill): assert "that skill owns validation" in t @pytest.mark.parametrize("skill", [API_SKILL, CLI_SKILL]) - def test_run_skills_read_the_round_from_make(self, skill): + def test_run_skills_confirm_the_benchmark_version(self, skill): t = _text(skill) + assert "make -s print-benchmark-version" in t assert "make -s print-pr-ref" in t assert "v0.6.0" not in t - - def test_make_exposes_the_current_round(self): - result = subprocess.run( - ["make", "-s", "print-pr-ref"], cwd=REPO_ROOT, - text=True, capture_output=True, check=True) - assert result.stdout.strip() + assert "problem-reductions-benchmark/blob/main/version" in t + assert "benchmark version:" in t + assert "wait for confirmation" in t + assert "checkout is outdated" in t + + def test_version_has_one_source_of_truth(self): + version = VERSION_FILE.read_text(encoding="utf-8").strip() + assert version.startswith("v") + for target in ("print-benchmark-version", "print-pr-ref"): + result = subprocess.run( + ["make", "-s", target], cwd=REPO_ROOT, + text=True, capture_output=True, check=True) + assert result.stdout.strip() == version + + @pytest.mark.parametrize("workflow", [ + REPO_ROOT / ".github/workflows/ci.yml", + REPO_ROOT / ".github/workflows/score-from-r2.yml", + REPO_ROOT / ".github/workflows/runner-image.yml", + ]) + def test_workflows_read_version_file(self, workflow): + text = workflow.read_text(encoding="utf-8") + assert "cat VERSION" in text + assert "PR_REF: v" not in text def test_router_sends_existing_results_to_submit_skill(self): t = _text(REPO_ROOT / ".agents/skills/run-benchmark/SKILL.md") From a46c7d13281af4af7a8d478f36ab764e01962f0f Mon Sep 17 00:00:00 2001 From: Xiwei Pan Date: Sun, 19 Jul 2026 20:36:20 +0800 Subject: [PATCH 4/4] Add benchmark version badge --- README.md | 2 ++ benchmark/tests/test_docs.py | 1 + 2 files changed, 3 insertions(+) diff --git a/README.md b/README.md index 4d3940a..f8a082f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Problem-Reductions Bug-Finding Benchmark +[![Benchmark version](https://img.shields.io/badge/benchmark-v0.6.0-blue)](VERSION) + A benchmark that measures how efficiently AI models find bugs in reduction rules from the [problem-reductions](https://github.com/CodingThrust/problem-reductions) library (290+ rules). The leaderboard is a static site (`site/`) published to **GitHub Pages**. Submitting uses a CLI (`python -m benchmark.submit`) that uploads your run to a private store; the maintainer re-verifies it with `pred` and publishes only the aggregate. See [CONTRIBUTING.md](CONTRIBUTING.md) to run and submit. diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index 4d14f1f..745f12f 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -78,6 +78,7 @@ def test_readme_has_metrics_section(self): def test_readme_lists_current_round_contract(self): text = README.read_text(encoding="utf-8") benchmark_version = VERSION_FILE.read_text(encoding="utf-8").strip() + assert f"badge/benchmark-{benchmark_version}-blue" in text assert f"[`{benchmark_version}`](VERSION)" in text assert PINNED_COMMIT in text assert f"`{PINNED_PRED_VERSION}`" in text